diff --git a/app/data/action/living_ui_actions.py b/app/data/action/living_ui_actions.py index a6366113..18388685 100644 --- a/app/data/action/living_ui_actions.py +++ b/app/data/action/living_ui_actions.py @@ -362,10 +362,14 @@ async def living_ui_list_projects(input_data: dict) -> dict: return {"status": "error", "message": "Living UI manager not initialized"} def _delivered(project_id: str) -> bool: + # Structural: an app with a live environment has been delivered + # (promoted or installed) — no stored flag to go stale. try: from app.factory.host_craftbot import get_factory_host as _gfh + from app.living_ui.lifecycle import has_live_env as _hle - return bool(_gfh().is_delivered(project_id)) + p = manager.projects.get(project_id) + return p is not None and _hle(p, _gfh()) except Exception: return False @@ -394,15 +398,15 @@ def _delivered(project_id: str) -> bool: @action( name="living_ui_notify_ready", description=( - "Launch or RELAUNCH a Living UI project: installs dependencies, runs the " - "validation gate, restarts backend and frontend, notifies the browser. " - "On a DELIVERED app it instead gates and boots a STAGING copy (cloned " - "disposable data, hidden port) and returns its URL — the user's live " - "app keeps running the previous version until walk_verify passes. " - "Call this ONLY after CREATING or CHANGING the app's CODE (migrations, " - "hooks, frontend). An app that is already running does NOT need it — " - "adding, editing or deleting DATA never requires a relaunch, and calling " - "it then rebuilds and restarts a live app for no reason. " + "Gate and boot a Living UI project's DEV environment after a CODE " + "change: installs dependencies, runs the validation gate, and serves " + "your new code on a hidden port with a FRESH empty database " + "(migrations replay at boot — no real data is ever cloned into it). " + "Returns the dev URL: test there. The user's live app (if any) keeps " + "running the previous version until walk_verify passes and promotes " + "the change. Call this ONLY after CREATING or CHANGING the app's CODE " + "(migrations, hooks, frontend) — adding, editing or deleting DATA " + "never requires it. EXTERNAL apps relaunch live instead (no dev env). " "Returns test errors if anything fails." ), default=False, @@ -462,32 +466,33 @@ async def living_ui_notify_ready(input_data: dict) -> dict: "message": "Living UI manager not initialized. Browser adapter may not be running.", } - # DELIVERED apps are gated and served in a STAGING copy: the gate's - # vite build overwrites the served pb_public in place, so running the - # normal pipeline on the real dir would blank the user's live UI — - # and testing against the real port would pollute real data. The - # live app keeps running the previous working version until - # walk_verify passes and flips it. EXTERNAL apps have no staging - # (nothing pb/-shaped to clone) — they always (re)launch live via - # their own pipeline. + # ONE flow for first builds and modifies: native apps are gated and + # served in a DEV environment — the project's code on a hidden port + # with a FRESH schema-only DB (migrations replay at boot; live data + # is never cloned). The live app (if any) keeps running the previous + # working version until walk_verify passes and PROMOTES the change. + # The gate's vite build overwrites the served pb_public in place, so + # running the pipeline on the real dir would blank a live UI — the + # dev copy also absorbs that. EXTERNAL apps have no dev env (nothing + # pb/-shaped) — they always (re)launch live via their own pipeline. _proj_pre = manager.get_project(project_id) _is_external = ( _proj_pre is not None and getattr(_proj_pre, "project_type", "native") == "external" ) - _is_delivered = False + _live_exists = False try: - from app.factory.host_craftbot import get_factory_host as _gfh + from app.living_ui.lifecycle import live_db_exists as _lde - _is_delivered = _gfh().is_delivered(project_id) + _live_exists = _proj_pre is not None and _lde(_proj_pre.path) except Exception: pass - if _is_delivered and not _is_external: - result = await manager.launch_staging(project_id) - else: - # Run the full pipeline: install → test → launch → verify + if _is_external: + # Run the external pipeline live: changes apply directly. result = await manager.launch_and_verify(project_id) + else: + result = await manager.open_dev(project_id) if result["status"] == "success": url = result.get("url", "") @@ -506,17 +511,26 @@ async def living_ui_notify_ready(input_data: dict) -> dict: get_factory_host().report_launch_success(project_id) except Exception: pass - staging_note = ( - "This is a STAGING copy with a disposable clone of the data — " - "the user's live app is untouched and still runs the previous " - "version; a passing walk_verify deploys your change to it. " - "Test freely against the staging URL. " - if _is_delivered and not _is_external + env_note = ( + ( + "This is the DEV environment: your new code with a FRESH, " + "empty database (migrations replayed — only data your " + "migrations seed exists; create any test records you " + "need). " + + ( + "The user's live app is untouched and still runs the " + "previous version; a passing walk_verify deploys your " + "change to it. " + if _live_exists + else "A passing walk_verify delivers the app to the " + "user with a clean database. " + ) + + "Test ONLY against this dev URL. " + ) + if not _is_external else ( "This EXTERNAL app runs live in its own runtime — changes " "apply directly; evidence is in logs/app.log. " - if _is_external and _is_delivered - else "" ) ) # Warn-only spec belt (LIFECYCLE-PLAN Phase 1): a modify whose @@ -524,7 +538,7 @@ async def living_ui_notify_ready(input_data: dict) -> dict: # stale contract — the verifier can't cover a change nobody # recorded. Never blocks a launch; everything here fails open. spec_note = "" - if _is_delivered and not _is_external and _proj_ok is not None: + if _live_exists and not _is_external and _proj_ok is not None: try: from pathlib import Path as _Path @@ -548,11 +562,19 @@ async def living_ui_notify_ready(input_data: dict) -> dict: ) except Exception: spec_note = "" + _dir_note = ( + f"Its files and logs are at {result.get('dir')} (read logs " + "THERE — your edits still go in the real project dir; " + "notify_ready syncs them in). " + if result.get("dir") + else "" + ) return { "status": "success", "message": ( f"App launched at {url} — gate, health and smoke checks " - f"passed. {staging_note}{spec_note}NOT VERIFIED YET: now call " + f"passed. {env_note}{_dir_note}{spec_note}NOT VERIFIED " + "YET: now call " f'living_ui_walk_verify(project_id="{project_id}") to run ' "the independent verifier against the running app. The " "build is complete ONLY when that returns success — do " @@ -608,19 +630,19 @@ async def living_ui_notify_ready(input_data: dict) -> dict: @action( name="living_ui_walk_verify", description=( - "Run the independent walk-verify sub-agent against the RUNNING Living " - "UI project: a real browser (headless) drives the app " + "Run the independent walk-verify sub-agent against the project's " + "DEV environment: a real browser (headless) drives the app " "feature-by-feature against reference/requirements.md. A clean " - "verdict announces the app to the user — the ONLY way a Living UI " - "BUILD completes. On a DELIVERED app it verifies the STAGING copy " - "(disposable data clone) and a clean verdict DEPLOYS the change to " - "the live app. Observed defects return the failure report: fix, " - "relaunch with living_ui_notify_ready, then call this again. " - "Requires living_ui_notify_ready first (it boots the app — or, for " - "a delivered app, its staging copy). " + "verdict PROMOTES the verified code to the live app (first build: " + "creates its live database fresh from migrations; update: applies " + "new migrations to the real data) and announces it — the ONLY way a " + "Living UI change completes. Observed defects return the failure " + "report: fix, relaunch with living_ui_notify_ready, then call this " + "again. Requires living_ui_notify_ready first (it boots the dev " + "env; external apps verify live instead). " "ONLY after building or modifying the app's CODE, never after a " "plain data change: it clicks through the UI creating test records " - "(isolated from the user's data, but pointless for data edits)." + "(in the dev env's disposable DB, but pointless for data edits)." ), default=False, mode="CLI", @@ -682,33 +704,30 @@ async def living_ui_walk_verify(input_data: dict) -> dict: if project is None: return {"status": "error", "message": f"Unknown project: {project_id}"} - # DELIVERED apps verify against their STAGING copy (disposable data - # clone on a hidden port) — never against the live app, whose DB - # holds real user data. `url` stays the REAL app's address: it is - # what gets announced after the flip. EXTERNAL apps have no staging - # (no pb_data to protect) — they always verify live and follow the - # build-mode branches (finalize is a safe no-op: no baseline). + # NATIVE apps always verify against their DEV environment (new code, + # fresh schema-only DB, hidden port) — never against the live app, + # whose DB holds real user data. `url` stays the REAL app's address: + # it is what gets announced after the promote. EXTERNAL apps have no + # dev env (no pb_data to protect) — they always verify live. _is_external = getattr(project, "project_type", "native") == "external" - _staging_record = None - try: - from app.factory.host_craftbot import get_factory_host as _gfh - - if not _is_external and _gfh().is_delivered(project_id): - _staging_record = _gfh().get_staging_record(project_id) - if not _staging_record: - return { - "status": "error", - "message": ( - "This app is delivered — verification runs against " - "a staging copy, and none exists. Call " - "living_ui_notify_ready first (it boots the " - "staging copy), then verify." - ), - } - except Exception: - _staging_record = None + _dev_record = None + if not _is_external: + try: + from app.factory.host_craftbot import get_factory_host as _gfh - if _staging_record is None and project.status != "running": + _dev_record = _gfh().get_staging_record(project_id) + except Exception: + _dev_record = None + if not _dev_record: + return { + "status": "error", + "message": ( + "Verification runs against the DEV environment, and " + "none exists. Call living_ui_notify_ready first (it " + "boots your code in the dev env), then verify." + ), + } + if _is_external and project.status != "running": return { "status": "error", "message": ( @@ -717,8 +736,8 @@ async def living_ui_walk_verify(input_data: dict) -> dict: ), } url = f"http://127.0.0.1:{project.port}" - verify_url = str(_staging_record.get("url")) if _staging_record else url - verify_path = str(_staging_record.get("dir")) if _staging_record else None + verify_url = str(_dev_record.get("url")) if _dev_record else url + verify_path = str(_dev_record.get("dir")) if _dev_record else None try: await broadcast_living_ui_progress( @@ -851,11 +870,12 @@ async def living_ui_walk_verify(input_data: dict) -> dict: } if kind == "defects": - # Observed misbehavior — the only thing that blocks a launch. - # Staging mode: the LIVE app runs the previous working version - # and stays up — availability wins; only the broken change (in - # the staging copy) is withheld. Build mode: stop as before. - if _staging_record is None: + # Observed misbehavior — the only thing that blocks a promote. + # Native: the LIVE app (if any) runs the previous working + # version and stays up — availability wins; only the broken + # change (in the dev env, which stays up for the fix mission) is + # withheld. External: stop the live app as before. + if _is_external: await manager.stop_project(project_id) defects = report.get("defects") or [] raw = (report.get("raw") or "")[:2500] @@ -873,9 +893,9 @@ async def living_ui_walk_verify(input_data: dict) -> dict: try: from pathlib import Path as _Path - # In staging mode the app under test wrote ITS OWN log — - # quoting the live app's log here would attribute the old - # version's lines to the new code. + # The dev instance under test wrote ITS OWN log — quoting + # the live app's log here would attribute the old version's + # lines to the new code. _log_root = str(verify_path or project.path) pb_log = _Path(_log_root) / "logs" / "pocketbase.log" # External apps log to app.log (their own runtime, no PB). @@ -938,10 +958,12 @@ async def living_ui_walk_verify(input_data: dict) -> dict: server_log=server_log, ) _stopped_note = ( - "The change was NOT deployed — the user's live app still " - "runs the previous working version. " - if _staging_record is not None - else "The app was stopped. " + "The app was stopped. " + if _is_external + else ( + "The change was NOT deployed — the user's live app still " + "runs the previous working version. " + ) ) if decision is None: # Machine done (a re-verify after delivery, outside a modify @@ -991,57 +1013,26 @@ async def living_ui_walk_verify(input_data: dict) -> dict: # announces to the user (FACTORY-PLAN §3.6 — no agent-authored # status); this run just ends. # - # Data-safety finalization comes FIRST, before any user-facing - # signal (plans/quizzical-greeting-alpaca): - # staging mode → FLIP: relaunch the real app with the verified - # code (migrations apply to real data at boot), destroy the - # staging copy and every test record in it. - # build mode → restore the pristine pb_data baseline so the - # user's first sight has no agent/verifier junk, then mark - # the app delivered. - if _staging_record is not None: - flip = await manager.finalize_modify(project_id) - if flip.get("status") != "success": - _flip_errors = flip.get("errors", []) - return { - "status": "error", - "message": ( - "Verification PASSED in staging, but deploying the " - "change to the live app failed at step " - f"'{flip.get('step', 'unknown')}'. The staging copy " - "was kept. Fix the errors below, then call " - "living_ui_notify_ready and living_ui_walk_verify " - "again." - ), - "test_errors": _flip_errors[:10], - } - else: - try: - from app.factory.host_craftbot import get_factory_host as _gfh2 - - _finalize = await manager.finalize_first_delivery(project_id) - if _finalize.get("status") != "success": - return { - "status": "error", - "message": ( - "Verification passed, but restoring the app to a " - "clean state for delivery failed at step " - f"'{_finalize.get('step', 'unknown')}'. Fix the " - "errors below, then call living_ui_notify_ready " - "and living_ui_walk_verify again." - ), - "test_errors": _finalize.get("errors", [])[:10], - } - _gfh2().mark_delivered(project_id) - except Exception as _fin_err: - # Delivery-state bookkeeping must never turn a verified app - # into a failure — worst case the app delivers as today - # (with test data) and stays in build mode. - import logging as _logging - - _logging.getLogger(__name__).warning( - f"[WALK_VERIFY] first-delivery finalize skipped: {_fin_err}" - ) + # PROMOTE comes FIRST, before any user-facing signal + # (docs/plans/living-ui-unified-lifecycle-plan.md): the real app + # boots with the verified code — pb_data absent (first delivery) → + # the migration chain creates it fresh; pb_data present (update) → + # new migrations apply on top and the data is otherwise untouched — + # then the dev env and every test record in it are destroyed. There + # is NO other path that touches a live database. + promoted = await manager.promote(project_id) + if promoted.get("status") != "success": + return { + "status": "error", + "message": ( + "Verification PASSED in the dev environment, but " + "deploying the change to the live app failed at step " + f"'{promoted.get('step', 'unknown')}'. The dev env was " + "kept. Fix the errors below, then call " + "living_ui_notify_ready and living_ui_walk_verify again." + ), + "test_errors": promoted.get("errors", [])[:10], + } await broadcast_living_ui_ready(project_id, url, project.port) if kind == "pass": @@ -1490,34 +1481,31 @@ def living_ui_http(input_data: dict) -> dict: "elapsed_ms": 0, "message": f"Project '{project_id}' not found.", } - # DELIVERED apps: while a staging copy exists, ALL agent/verifier HTTP - # goes to it — this action resolves the REAL app's port on its own, and - # without the redirect a staging-mode verifier would write test records - # straight into real user data through this side door. With NO staging - # copy, intent decides: mid-arc (factory machine non-terminal — a code - # change is being built) a mutating call is agent test traffic and is - # refused toward staging; arc closed (machine terminal) it is normal - # OPERATION of the delivered app — the write IS user data ("add this - # lead for me") and belongs in the live app. Refusing those too routed - # real records into the disposable staging clone, where the deploy flip - # destroys them (observed live 2026-08-05, RBS Leads Tracker). - _staging_url = None - _is_delivered = False + # While a DEV environment exists, ALL agent/verifier HTTP goes to it — + # this action resolves the REAL app's port on its own, and without the + # redirect a verifier would write test records straight into real user + # data through this side door. With NO dev env, intent decides: mid-arc + # (factory machine non-terminal — a code change is being built) a + # mutating call is agent test traffic and is refused toward the dev env; + # arc closed (machine terminal) it is normal OPERATION of the app — the + # write IS user data ("add this lead for me") and belongs in the live + # app. Refusing those too routed real records into the disposable dev + # copy, where the promote destroys them (observed live 2026-08-05, RBS + # Leads Tracker). + _dev_url = None _mid_arc = False try: from app.factory.host_craftbot import get_factory_host as _gfh - _is_delivered = _gfh().is_delivered(project_id) - if _is_delivered: - _rec = _gfh().get_staging_record(project_id) - if _rec and _rec.get("url"): - _staging_url = str(_rec["url"]) - _machine = _gfh().machine_for(project_id) - _mid_arc = _machine is not None and not _machine.terminal + _rec = _gfh().get_staging_record(project_id) + if _rec and _rec.get("url"): + _dev_url = str(_rec["url"]) + _machine = _gfh().machine_for(project_id) + _mid_arc = _machine is not None and not _machine.terminal except Exception: - _staging_url = None + _dev_url = None - if _is_delivered and not _staging_url and _mid_arc and method != "GET": + if _dev_url is None and _mid_arc and method != "GET": return { "status": "error", "status_code": 0, @@ -1526,17 +1514,17 @@ def living_ui_http(input_data: dict) -> dict: "final_url": "", "elapsed_ms": 0, "message": ( - f"Project '{project_id}' is delivered and a code change is in " - "progress — its data is real user data, and agent test writes " - "outside a staging copy are refused. For the code change, " - "call living_ui_notify_ready first (it boots the staging " - "copy), then retry against it. If you meant to store REAL " + f"A code change is in progress for project '{project_id}' — " + "its live data is real user data, and agent test writes " + "outside the dev environment are refused. For the code " + "change, call living_ui_notify_ready first (it boots the dev " + "env), then retry against it. If you meant to store REAL " "data the user asked for, wait until the change arc finishes " "— live data writes resume then." ), } - if _staging_url is None and project.status != "running": + if _dev_url is None and project.status != "running": return { "status": "error", "status_code": 0, @@ -1547,9 +1535,7 @@ def living_ui_http(input_data: dict) -> dict: "message": f"Project '{project_id}' is not running (status: {project.status}). Launch it first.", } - base_url = _staging_url or ( - project.backend_url if target == "backend" else project.url - ) + base_url = _dev_url or (project.backend_url if target == "backend" else project.url) if not base_url: # Fall back to constructing from port if URL field is missing port = project.backend_port if target == "backend" else project.port @@ -1626,13 +1612,13 @@ def living_ui_http(input_data: dict) -> dict: # If the agent just mutated the Living UI's data, tell the browser so the # iframe reloads to show fresh state. The frontend debounces these so a - # burst of writes only triggers one reload. Staging writes hit the + # burst of writes only triggers one reload. Dev-env writes hit the # disposable copy — the user's iframe shows the LIVE app, so a reload # would be noise about data it can't even see. if ( resp.ok and method in {"POST", "PUT", "PATCH", "DELETE"} - and _staging_url is None + and _dev_url is None ): try: from app.living_ui import dispatch_living_ui_data_changed @@ -1993,12 +1979,12 @@ async def living_ui_marketplace_install(input_data: dict) -> dict: # ADOPT the current build session's project instead of minting a # duplicate: a wizard-created project already owns the tab, port and - # session this run lives in. Only never-delivered scaffolds are - # adopted — a DELIVERED session project means the user is installing - # a separate new app, which stays a fresh project. (Observed live - # 2026-08-05: installing without adoption left an orphan project - # whose factory machine redispatched a from-scratch build of the - # same app.) + # session this run lives in. Only undelivered scaffolds are adopted + # — a project with a live database (or an already-installed + # marketplace app) means the user is installing a separate new app, + # which stays a fresh project. (Observed live 2026-08-05: installing + # without adoption left an orphan project whose factory machine + # redispatched a from-scratch build of the same app.) adopt_id = None _sid = str(input_data.get("_session_id") or "") if _sid.startswith("lui_"): @@ -2007,9 +1993,19 @@ async def living_ui_marketplace_install(input_data: dict) -> dict: if _proj is not None and _proj.path: _delivered = False try: - from app.factory.host_craftbot import get_factory_host as _gfh + import json as _json2 + from pathlib import Path as _P2 + + from app.living_ui.lifecycle import live_db_exists as _lde - _delivered = _gfh().is_delivered(_candidate) + _delivered = _lde(_proj.path) + if not _delivered: + _mf2 = _json2.loads( + (_P2(str(_proj.path)) / "manifest.json").read_text( + encoding="utf-8" + ) + ) + _delivered = bool(_mf2.get("marketplaceAppId")) except Exception: _delivered = False if not _delivered: @@ -2115,10 +2111,10 @@ async def living_ui_marketplace_install(input_data: dict) -> dict: "message": ( f"Marketplace app '{app_id}' installed into this project " f"at {url}. NOT DONE: now apply ONLY the adaptations " - "listed in reference/requirements.md (the app counts as " - "delivered, so living_ui_notify_ready will boot a staging " - "copy), then living_ui_walk_verify to deploy and " - "announce. If the requirements list no concrete " + "listed in reference/requirements.md " + "(living_ui_notify_ready will boot your changes in the " + "dev environment), then living_ui_walk_verify to deploy " + "and announce. If the requirements list no concrete " "adaptations, ask the user what to change with a final " "send_message instead of guessing." + _triggers_brief ), diff --git a/app/data/agent_file_system_template/AGENT.md b/app/data/agent_file_system_template/AGENT.md index 28675368..db8d39e1 100644 --- a/app/data/agent_file_system_template/AGENT.md +++ b/app/data/agent_file_system_template/AGENT.md @@ -1159,17 +1159,19 @@ living_ui_scaffold(name, description, ...) Create a project: copies the bluepri living_ui_list_projects() {id, name, description, status, url, path, delivered}. Resolve "the app" to an id here, never by filesystem search. living_ui_notify_ready(project_id) Launch pipeline: install deps → validation gate (types, - build, migrations, ops manifest) → boot PocketBase + - frontend → health check. On a delivered app it boots a - STAGING copy (cloned data, hidden port), never the live app. - Gate failures come back in test_errors. Circuit breaker: - identical error ×3 warns, ×6 stops. -living_ui_walk_verify(project_id) Headless-browser sub-agent drives the running app + build, migrations, ops manifest) → boot the DEV environment + (your code on a hidden port with a FRESH schema-only DB — + migrations replay; live data is never cloned). The live app + (if any) keeps running untouched. Gate failures come back + in test_errors. Circuit breaker: identical error ×3 warns, + ×6 stops. +living_ui_walk_verify(project_id) Headless-browser sub-agent drives the DEV instance feature-by-feature against reference/requirements.md. Verdicts: pass | incomplete | defects | blocked | unparseable. - A clean pass is the ONLY way a build completes: first build - → project marked delivered; delivered app → staging flips - to live. 35-minute ceiling. + A clean pass is the ONLY way a change completes: it PROMOTES + the code to the live app (first build → live DB created + fresh from migrations; update → new migrations apply to the + real data) and destroys the dev copy. 35-minute ceiling. living_ui_restart(project_id) Stop + full launch pipeline. living_ui_report_progress(project_id, ...) Creation-phase progress. No-op once the project runs. living_ui_usage(project_id) Returns the project's operating manual: path, live data @@ -1200,15 +1202,17 @@ node /living-ui/tools/src/cli.ts run -- node /living-ui/tools/src/cli.ts ops ``` -`living_ui_usage(project_id)` returns the exact commands for a given project. Use `living_ui_http` only when the CLI cannot do it. Writes to a delivered app's real data outside a staging arc are refused. +`living_ui_usage(project_id)` returns the exact commands for a given project. Use `living_ui_http` only when the CLI cannot do it. While a code change is in progress, agent writes are routed to the dev instance — test writes to an app's real data are refused. -### Build / delivery lifecycle +### Build / delivery lifecycle (one flow for builds and modifies) ``` -scaffold → dedicated build session writes code → notify_ready (validation gate + boot) - → walk_verify pass → delivered (live URL announced by the factory host) -modify a delivered app → changes go to a STAGING clone on a hidden port - → notify_ready boots staging → walk_verify pass → staging flips to live +write code in the project dir → notify_ready (validation gate + boot of the + DEV env: code copy, hidden port, fresh schema-only DB) + → walk_verify drives the dev instance → clean pass PROMOTES: + live app boots the new code (first build: live DB created fresh from + migrations; update: new migrations apply to real data), dev copy + destroyed, ready announced by the factory host ``` - The factory host owns retries, fix-mission dispatch, and the "ready" announcement. Do not author success status messages for a build yourself. diff --git a/app/factory/host_craftbot.py b/app/factory/host_craftbot.py index 395d09ac..c2ce21c0 100644 --- a/app/factory/host_craftbot.py +++ b/app/factory/host_craftbot.py @@ -104,23 +104,20 @@ def _sidecar_write(self, project_id: str, data: Dict[str, Any]) -> None: except Exception as e: logger.debug(f"[FACTORY] sidecar write failed: {e}") - # ── delivery lifecycle (sidecar-backed; see plans/quizzical-greeting) ── - # "delivered" picks the data-safety mode for every later gate/verify: - # not delivered → the DB is disposable (verify live, restore the pristine - # baseline before announcing); delivered → real user data, everything runs - # in a staging copy. machine.terminal is NOT a substitute predicate: - # STUCK is terminal too, and marketplace/ZIP installs never get a machine. - def is_delivered(self, project_id: str) -> bool: - return bool(self._sidecar_read(project_id).get("delivered")) - - def mark_delivered(self, project_id: str) -> None: + # ── delivery bookkeeping (sidecar-backed) ────────────────────────────── + # delivered_at is a COSMETIC timestamp (requirements-staleness warning, + # announce wording) — never a control input. The retired "delivered" + # flag used to pick the data-safety mode and went stale on real apps + # (2026-08-19: a two-week-in-use CRM read as never-delivered and its + # live DB was wiped by the first-delivery baseline restore). Every + # lifecycle predicate is now structural: lifecycle.live_db_exists(). + def stamp_delivered(self, project_id: str) -> None: side = self._sidecar_read(project_id) - if side.get("delivered"): + if side.get("delivered_at"): return - side["delivered"] = True side["delivered_at"] = time.time() self._sidecar_write(project_id, side) - logger.info(f"[FACTORY] {project_id} marked delivered") + logger.info(f"[FACTORY] {project_id} delivery stamped") # ── trigger-plane consent (spec TRIGGERS-PLAN) ───────────────────────── # An app that can fire the agent can drive a session holding the user's @@ -246,9 +243,38 @@ def delivered_at(self, project_id: str) -> Optional[float]: except (TypeError, ValueError): return None + # ── backup bookkeeping (sidecar-backed; spec living-ui-backups-plan) ─── + # last_at drives the scheduler's due check (absent -> due now, which is + # also the catch-up-after-restart path); last_error is surfaced on the + # settings card and cleared by the next success. + def record_backup_ok(self, project_id: str, ts: float) -> None: + side = self._sidecar_read(project_id) + side["backup"] = {"last_at": float(ts)} + self._sidecar_write(project_id, side) + + def record_backup_error(self, project_id: str, message: str) -> None: + side = self._sidecar_read(project_id) + state = side.get("backup") + state = dict(state) if isinstance(state, dict) else {} + state["last_error"] = str(message)[:500] + side["backup"] = state + self._sidecar_write(project_id, side) + + def backup_state(self, project_id: str) -> Dict[str, Any]: + """{"last_at": float|None, "last_error": str|None} — always both keys.""" + state = self._sidecar_read(project_id).get("backup") + state = state if isinstance(state, dict) else {} + try: + last_at = ( + float(state["last_at"]) if state.get("last_at") is not None else None + ) + except (TypeError, ValueError): + last_at = None + return {"last_at": last_at, "last_error": state.get("last_error") or None} + def begin_modify(self, project_id: str) -> None: - """A modify of a delivered app is starting (called from - launch_staging success — deterministic, never agent-dependent): + """A modify of an app with a live database is starting (called from + open_dev success — deterministic, never agent-dependent): re-arm the machine into MODIFYING so the whole supervision apparatus (fix missions, caps, stuck reports, announcements) applies to the modify exactly as it did to the build (LIFECYCLE-PLAN Phase 2). @@ -257,7 +283,7 @@ def begin_modify(self, project_id: str) -> None: VIRGIN (no history — machine_for mints BUILDING for marketplace/ imported apps that never had an arc). A non-terminal machine WITH history means a modify/fix arc is already in flight — a fix - mission's notify_ready re-enters launch_staging — so no-op. + mission's notify_ready re-enters open_dev — so no-op. """ machine = self.machine_for(project_id) if machine is None: @@ -277,9 +303,10 @@ def begin_modify(self, project_id: str) -> None: f"(generation {machine.generation})" ) - # The staging record is the single source of truth for "a staging copy of - # this app exists": actions redirect to it, the reaper kills from it, and - # clearing it is what ends staging mode. + # The staging record is the single source of truth for "a dev environment + # of this app exists": actions redirect to it, the reaper kills from it, + # and clearing it is what ends dev mode. (Key name "staging" is + # historical — kept so records from older versions stay readable.) def get_staging_record(self, project_id: str) -> Optional[Dict[str, Any]]: record = self._sidecar_read(project_id).get("staging") return record if isinstance(record, dict) else None @@ -516,6 +543,15 @@ def _compose_fix_brief( if books else "" ) + # The RUNNING instance is the dev environment when one is up — + # repro commands and logs must target it, not the (possibly not even + # running) live project dir. + _dev_rec = self.get_staging_record(project.id) + run_dir = ( + str(_dev_rec.get("dir")) + if _dev_rec and _dev_rec.get("dir") + else str(project.path) + ) return f"""FIX MISSION {n} for Living UI '{project.name}' ({project.id}). The independent verifier drove the app in a real browser. Each DEFECT below @@ -526,12 +562,14 @@ def _compose_fix_brief( {books_text} === HOW TO WORK (concrete) === -1. Reproduce first: use the repro commands / exercise the failing op: - {cli} run {project.path} -2. Read the evidence before theorizing: {project.path}/logs/pocketbase.log +1. Reproduce first: use the repro commands / exercise the failing op + against the RUNNING dev instance: + {cli} run {run_dir} +2. Read the evidence before theorizing: {run_dir}/logs/pocketbase.log (every causal claim must quote a log line; if you can't quote it, gather more evidence — "unknown, investigating" is valid, a guess is not). -3. Fix in {project.path} (hooks/migrations/frontend per the ownership rules). +3. Fix in {project.path} (hooks/migrations/frontend per the ownership rules) + — living_ui_notify_ready syncs your edits into the dev instance. 4. Relaunch: living_ui_notify_ready(project_id="{project.id}") 5. Verify: living_ui_walk_verify(project_id="{project.id}") The system tracks attempts and reports status to the user — do NOT send @@ -565,16 +603,20 @@ def _emit_mission( mission_id = f"{mission_kind}-{int(time.time())}" # Modify-era missions (a reopened machine) get the modify skill — - # staging semantics and the never-touch-pb_data rules live there; + # dev-env semantics and the never-touch-pb_data rules live there; # build-era missions keep the full creator workflow. A machine - # re-armed from a stuck BUILD (never delivered — no user data to - # protect) is still build-era despite generation > 0; a stuck - # MODIFY of a delivered app keeps the modify skill. + # re-armed from a stuck BUILD (no live database yet — no user data + # to protect) is still build-era despite generation > 0; a stuck + # MODIFY of an app with live data keeps the modify skill. gens = machine.generations() + try: + from app.living_ui.lifecycle import has_live_env + + _has_live = has_live_env(project, self) + except Exception: + _has_live = False resumed_stuck_build = ( - bool(gens) - and gens[-1].get("final_state") == STUCK - and not self.is_delivered(project.id) + bool(gens) and gens[-1].get("final_state") == STUCK and not _has_live ) workflow_skill = ( "living-ui-modify" diff --git a/app/living_ui/integration_bridge.py b/app/living_ui/integration_bridge.py index 5ccf612f..87311082 100644 --- a/app/living_ui/integration_bridge.py +++ b/app/living_ui/integration_bridge.py @@ -479,21 +479,23 @@ async def _handle_agent_request(self, request: web.Request) -> web.Response: status=403, ) - # Gate 4 — era. Pre-delivery fires are agent/verifier test - # traffic (the walk verifier clicks ⚡ buttons), and a staging - # copy aliases to the real project id through the shared bridge - # token. Neither may start real agent runs — pre-delivery rows - # are wiped by the baseline restore anyway. NOT keyed on the - # factory machine: machine_for lazily creates a BUILDING machine - # for any project, so a marketplace install (which never builds - # here) would read as mid-arc forever. - delivered = host.is_delivered(project_id) - staging = host.get_staging_record(project_id) - if not delivered or staging: + # Gate 4 — era. While a DEV environment exists, fires are + # agent/verifier test traffic (the walk verifier clicks ⚡ + # buttons in the dev instance, which aliases to the real project + # id through the shared bridge token) — they must not start real + # agent runs; the dev copy and its rows die at promote anyway. + # With no dev env there is nothing mid-change that could fire + # falsely: during a first build the live app does not run yet, + # and after a promote fires are legitimate operation. NOT keyed + # on the factory machine: machine_for lazily creates a BUILDING + # machine for any project, so a marketplace install (which never + # builds here) would read as mid-arc forever. + dev_env = host.get_staging_record(project_id) + if dev_env: logger.info( f"[INTEGRATION_BRIDGE] trigger fire deferred (era) " f"project={project_id} trigger={trigger!r} " - f"delivered={delivered} staging={bool(staging)}" + f"dev_env=True" ) return web.json_response( { diff --git a/app/living_ui/lifecycle/__init__.py b/app/living_ui/lifecycle/__init__.py new file mode 100644 index 00000000..5d39ff7a --- /dev/null +++ b/app/living_ui/lifecycle/__init__.py @@ -0,0 +1,47 @@ +"""Unified Living UI lifecycle — dev/live environment separation. + +One flow for first builds and modifies (spec: +docs/plans/living-ui-unified-lifecycle-plan.md, from the 2026-08-19 CRM +data-loss incident): every code change is developed and verified in a DEV +environment — a runtime copy of the project's code booted on a hidden port +with a FRESH, schema-only database (the migration chain replays at boot; +live data is never cloned). A clean verify PROMOTES: the real project boots +with the new code, new migrations apply to the real pb_data, and the dev +copy is destroyed. + +The single invariant this package enforces: + + Nothing writes to a live environment's pb_data except (a) PocketBase's + migration replay during Promoter.promote(), and (b) a USER-CONFIRMED + restore of a backup archive (manager.restore_backup, spec + docs/plans/living-ui-backups-requirements.md FR9 — reversible by + design: the pre-restore state is captured first, and the restore + aborts if that capture fails). The agent has no restore action. + +There is no stored "delivered" mode flag — the one thing it used to decide +(first vs update promote) is derived from filesystem state via +live_db_exists(), which cannot go stale the way the sidecar flag did. +""" + +from app.living_ui.lifecycle.backups import BackupEntry, BackupService, BackupStore +from app.living_ui.lifecycle.environment import ( + DevInstance, + has_live_env, + live_db_exists, +) +from app.living_ui.lifecycle.lifecycle import AppLifecycle +from app.living_ui.lifecycle.promoter import Promoter +from app.living_ui.lifecycle.provisioner import DEV_PORT_RANGE, DevProvisioner + +__all__ = [ + "AppLifecycle", + "BackupEntry", + "BackupService", + "BackupStore", + "DevInstance", + "DevProvisioner", + "DEV_PORT_RANGE", + "Promoter", + "has_live_env", + "live_db_exists", +] diff --git a/app/living_ui/lifecycle/backups.py b/app/living_ui/lifecycle/backups.py new file mode 100644 index 00000000..a237cc8a --- /dev/null +++ b/app/living_ui/lifecycle/backups.py @@ -0,0 +1,339 @@ +"""BackupStore + BackupService — backups of a Living UI app's live pb_data. + +Spec: docs/plans/living-ui-backups-requirements.md (+ -plan.md). One archive +format for every trigger: a ZIP of the snapshot layout snapshot_pb_data +produces (every *.db consistent via sqlite's backup API + storage/), named +__.zip under living_ui/_backups// — OUTSIDE the +project dir, so it survives anything that deletes or restores the project's +own pb_data (the 2026-08-19 incident class this feature answers). + +Two capture paths, one output: + - capture_stopped: sync, snapshot_pb_data + zip. Also correct while the + app RUNS for the DB half (sqlite backup API tolerates a live writer) — + it is the pre-promote / pre-restore path, where a fs-level skew between + DB and storage/ is acceptable and no event loop is guaranteed. + - capture_running: PocketBase's own POST /api/backups (superuser-authed) — + the only DB+files-ATOMIC option (PB goes read-only for the duration) — + then the finished zip is MOVED out of pb_data/backups/ into the store. + +Deletion discipline: everything goes through _guarded_delete, which requires +a strict descendant of the _backups root (same predicate as +DevProvisioner._guarded_rmtree / pb_data_io). prune()/delete() additionally +touch only filenames matching the canonical pattern — files we cannot +attribute to a pool are never deleted (FR5). +""" + +import re +import shutil +import time +import zipfile +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import List, Optional + +try: + from loguru import logger +except ImportError: + import logging + + logger = logging.getLogger(__name__) + +from app.living_ui.pb_data_io import snapshot_pb_data + +TRIGGERS = ("scheduled", "pre_promote", "manual") + +# pre_promote-pool retention — a deliberate constant, not a setting (resolved +# question §7.2 in the requirements: a second retention knob on the card +# requires understanding what a promote is; the scheduled pool owns depth). +PRE_PROMOTE_KEEP = 3 + +# __.zip — the trigger token doubles as the retention pool. +_NAME_RE = re.compile( + r"^(?P\d{8}T\d{6}Z)__(?Pscheduled|pre_promote|manual)\.zip$" +) +# Same guard the provisioner/wizard use: nothing outside this pattern ever +# becomes part of a deleted path. +_ID_RE = re.compile(r"^[A-Za-z0-9_-]{4,64}$") + + +@dataclass +class BackupEntry: + project_id: str + path: Path + ts: float # epoch, UTC + trigger: str + size: int + + @property + def filename(self) -> str: + return self.path.name + + +def _ts_name(ts: float) -> str: + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def _parse_ts(token: str) -> float: + return ( + datetime.strptime(token, "%Y%m%dT%H%M%SZ") + .replace(tzinfo=timezone.utc) + .timestamp() + ) + + +class BackupStore: + """Layout, listing and pool-aware pruning under living_ui/_backups/. + Pure filesystem — knows nothing about projects beyond their id.""" + + def __init__(self, living_ui_dir: Path) -> None: + self.living_ui_dir = Path(living_ui_dir) + self.root = self.living_ui_dir / "_backups" + + # ── layout ───────────────────────────────────────────────────────────── + def project_dir(self, project_id: str) -> Path: + if not _ID_RE.match(project_id or ""): + raise ValueError(f"unsafe project id for backups: {project_id!r}") + return self.root / project_id + + def claim_path( + self, project_id: str, trigger: str, ts: Optional[float] = None + ) -> Path: + """Reserve a canonical archive path (parent created, name unique — + same-second collisions bump the timestamp forward).""" + if trigger not in TRIGGERS: + raise ValueError(f"unknown backup trigger: {trigger!r}") + pdir = self.project_dir(project_id) + pdir.mkdir(parents=True, exist_ok=True) + ts = time.time() if ts is None else ts + path = pdir / f"{_ts_name(ts)}__{trigger}.zip" + while path.exists(): + ts += 1 + path = pdir / f"{_ts_name(ts)}__{trigger}.zip" + return path + + # ── listing ──────────────────────────────────────────────────────────── + def list_backups(self, project_id: str) -> List[BackupEntry]: + """All attributable archives for the project, newest first. Files + not matching the canonical name are invisible here (and therefore + untouchable by prune/delete).""" + pdir = self.project_dir(project_id) + entries: List[BackupEntry] = [] + if not pdir.is_dir(): + return entries + for f in pdir.iterdir(): + m = _NAME_RE.match(f.name) + if not m or not f.is_file(): + continue + entries.append( + BackupEntry( + project_id=project_id, + path=f, + ts=_parse_ts(m.group("ts")), + trigger=m.group("trigger"), + size=f.stat().st_size, + ) + ) + entries.sort(key=lambda e: e.ts, reverse=True) + return entries + + def total_size(self, project_id: str) -> int: + return sum(e.size for e in self.list_backups(project_id)) + + def orphan_dirs(self, registered_ids) -> List[str]: + """Backup dirs whose project no longer exists (D5: listed for manual + cleanup, never auto-reaped).""" + if not self.root.is_dir(): + return [] + known = set(registered_ids) + return sorted( + d.name for d in self.root.iterdir() if d.is_dir() and d.name not in known + ) + + # ── deletion ─────────────────────────────────────────────────────────── + def prune(self, project_id: str, trigger: str, keep: int) -> int: + """Delete the oldest archives of one pool beyond `keep`. Other pools + and unattributable files are untouched.""" + if trigger not in TRIGGERS: + raise ValueError(f"unknown backup trigger: {trigger!r}") + keep = max(0, int(keep)) + pool = [e for e in self.list_backups(project_id) if e.trigger == trigger] + doomed = pool[keep:] # list is newest-first + for entry in doomed: + self._guarded_delete(entry.path) + if doomed: + logger.info( + f"[LIVING_UI:BACKUP] pruned {len(doomed)} {trigger} backup(s) " + f"of {project_id} (keep {keep})" + ) + return len(doomed) + + def delete(self, project_id: str, filename: str) -> None: + """Delete one archive by its canonical filename (user-driven).""" + if not _NAME_RE.match(filename or ""): + raise ValueError(f"not a backup archive name: {filename!r}") + self._guarded_delete(self.project_dir(project_id) / filename) + + def delete_project_backups(self, project_id: str) -> None: + """Remove the project's whole backup dir (delete-project opt-in, D5).""" + pdir = self.project_dir(project_id) + if pdir.exists(): + self._guarded_delete(pdir) + + def _guarded_delete(self, target: Path) -> None: + """Only ever delete strictly inside the _backups root.""" + resolved = Path(target).resolve() + root = self.root.resolve() + if root not in resolved.parents: + raise ValueError(f"refusing to delete {resolved} — outside {root}") + if resolved.is_dir(): + shutil.rmtree(resolved) + else: + resolved.unlink() + + +class BackupService: + """Capture orchestration. Composed by the manager (like the lifecycle); + never reaches back into registry, sessions or broadcasting.""" + + def __init__(self, living_ui_dir: Path) -> None: + self.living_ui_dir = Path(living_ui_dir) + self.store = BackupStore(living_ui_dir) + + # ── stopped / hook path ──────────────────────────────────────────────── + def capture_stopped(self, project, trigger: str) -> BackupEntry: + """snapshot_pb_data + zip. Correct with the app stopped (consistent + by absence of writers) and acceptable while it runs (DBs consistent + via the sqlite backup API; storage/ may skew by the copy window) — + the pre-promote and pre-restore path. Raises on failure; callers + decide fatality (scheduler: log+retry; pre-promote hook: abort).""" + final = self.store.claim_path(project.id, trigger) + tmp_root = final.parent / ".tmp" + if tmp_root.exists(): + self.store._guarded_delete(tmp_root) # crashed prior capture + snapshot = tmp_root / "pb_data" + try: + snapshot_pb_data( + Path(project.path) / "pb" / "pb_data", snapshot, self.living_ui_dir + ) + self._zip_dir(snapshot, final) + finally: + if tmp_root.exists(): + self.store._guarded_delete(tmp_root) + entry = BackupEntry( + project_id=project.id, + path=final, + ts=_parse_ts(_NAME_RE.match(final.name).group("ts")), + trigger=trigger, + size=final.stat().st_size, + ) + logger.info( + f"[LIVING_UI:BACKUP] {project.id} {trigger} backup (stopped path) " + f"-> {final.name} ({entry.size} bytes)" + ) + return entry + + # ── running path ─────────────────────────────────────────────────────── + async def capture_running(self, project, trigger: str) -> BackupEntry: + """PocketBase's own backup API: one ATOMIC zip of pb_data (DB + + storage; PB goes read-only for the duration), moved out of + pb_data/backups/ into the store. Raises on any failure — never + falls back to a raw copy of a live DB (FR2).""" + from app.living_ui.runner import read_superuser_creds + + creds = read_superuser_creds(Path(project.path)) + if creds is None: + raise RuntimeError( + f"no .superuser credentials for {project.id} — cannot call " + "the PocketBase backup API" + ) + email, password = creds + base = f"http://127.0.0.1:{project.port}" + # PB restricts backup names to [a-z0-9_-].zip — use a throwaway name + # and let the store rename impose the canonical one. + pb_name = f"craftbot_{int(time.time())}.zip" + + import aiohttp + + timeout = aiohttp.ClientTimeout(total=300) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post( + f"{base}/api/collections/_superusers/auth-with-password", + json={"identity": email, "password": password}, + ) as resp: + if resp.status != 200: + raise RuntimeError(f"superuser auth failed ({resp.status})") + token = (await resp.json()).get("token") or "" + async with session.post( + f"{base}/api/backups", + json={"name": pb_name}, + headers={"Authorization": token}, + ) as resp: + if resp.status not in (200, 204): + body = (await resp.text())[:300] + raise RuntimeError( + f"PocketBase backup failed ({resp.status}): {body}" + ) + + produced = Path(project.path) / "pb" / "pb_data" / "backups" / pb_name + if not produced.exists(): + raise RuntimeError(f"PocketBase reported success but {pb_name} is missing") + final = self.store.claim_path(project.id, trigger) + shutil.move(str(produced), str(final)) + entry = BackupEntry( + project_id=project.id, + path=final, + ts=_parse_ts(_NAME_RE.match(final.name).group("ts")), + trigger=trigger, + size=final.stat().st_size, + ) + logger.info( + f"[LIVING_UI:BACKUP] {project.id} {trigger} backup (PB API) " + f"-> {final.name} ({entry.size} bytes)" + ) + return entry + + # ── restore support ──────────────────────────────────────────────────── + def prepare_restore(self, entry: BackupEntry) -> Path: + """Unzip an archive to a temp dir under the guard root and validate + it — the snapshot-layout dir restore_pb_data expects. The caller + (manager.restore_backup) owns stop/replace/relaunch; this keeps all + archive handling in one module. Caller must remove the returned dir + (it lives under _backups//.restore-tmp, so the next prepare also + sweeps a leftover).""" + target = self.store.project_dir(entry.project_id) / ".restore-tmp" + if target.exists(): + self.store._guarded_delete(target) + with zipfile.ZipFile(entry.path) as zf: + for name in zf.namelist(): + # Belt against hostile archives: no absolute paths, no + # parent-dir escapes (the store only holds files we wrote, + # but an uploaded/copied-in zip costs one loop to distrust). + p = Path(name) + if p.is_absolute() or ".." in p.parts: + raise ValueError(f"unsafe path in archive: {name!r}") + zf.extractall(target) + if not (target / "data.db").exists(): + self.store._guarded_delete(target) + raise ValueError(f"{entry.filename} has no data.db — not a pb_data backup") + return target + + def cleanup_restore(self, entry: BackupEntry) -> None: + target = self.store.project_dir(entry.project_id) / ".restore-tmp" + if target.exists(): + self.store._guarded_delete(target) + + # ── internals ────────────────────────────────────────────────────────── + def _zip_dir(self, src_dir: Path, final: Path) -> None: + """Zip src_dir's CONTENTS to `final` — written as a sibling .tmp and + renamed into place, so a partial archive is never listable.""" + tmp = final.with_name(final.name + ".part") + try: + with zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED) as zf: + for f in sorted(src_dir.rglob("*")): + if f.is_file(): + zf.write(f, f.relative_to(src_dir)) + tmp.replace(final) + finally: + if tmp.exists(): + tmp.unlink() diff --git a/app/living_ui/lifecycle/environment.py b/app/living_ui/lifecycle/environment.py new file mode 100644 index 00000000..7d180152 --- /dev/null +++ b/app/living_ui/lifecycle/environment.py @@ -0,0 +1,85 @@ +"""Environment identity: the dev-instance value object and the one +structural predicate the lifecycle branches on. + +live_db_exists() replaces the retired "delivered" sidecar flag. The flag +could diverge from reality (it did, 2026-08-19: a two-week-in-use CRM read +as never-delivered and its live DB was restored to a stale baseline); the +filesystem cannot — a live database either exists or it does not. +""" + +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Optional, Union + + +def live_db_exists(project_path: Union[str, Path]) -> bool: + """True when the project's LIVE environment has a real database. + + This is the first-vs-update promote predicate: absent -> the promote + boot creates pb_data fresh from the migration chain (first delivery); + present -> the boot applies only new migrations on top and the data is + otherwise untouched. External apps have no pb/ shape and never match — + ask has_live_env() when the project might be one. + """ + try: + return (Path(project_path) / "pb" / "pb_data" / "data.db").exists() + except (TypeError, OSError): + return False + + +def has_live_env(project, host) -> bool: + """True when `project` has a LIVE environment to protect — the one + build-vs-modify predicate for callers that may hold an external app. + + Native apps answer structurally (live_db_exists); external apps have no + pb/ shape, so the nearest structural fact is whether a promote ever + succeeded (host.delivered_at — a write-once timestamp, not the retired + mode flag). `host` is passed in, never imported: this module stays + import-clean below the factory host. + """ + if getattr(project, "project_type", "native") == "external": + return host.delivered_at(project.id) is not None + return live_db_exists(project.path) + + +@dataclass +class DevInstance: + """One dev environment: the project's code copied to a hidden port with + its own (fresh) database. `process` is runtime-only; everything else + round-trips through the factory-host sidecar record. + + The sidecar key and on-disk root keep their historical "staging" names — + they are storage details shared with records written by older versions, + and the boot reaper must keep finding both. + """ + + project_id: str + dir: Path + port: int + created_at: float + pid: Optional[int] = None + process: Optional[subprocess.Popen] = None + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def to_record(self) -> Dict[str, Any]: + return { + "dir": str(self.dir), + "port": self.port, + "url": self.url, + "pid": self.pid, + "created_at": self.created_at, + } + + @classmethod + def from_record(cls, project_id: str, record: Dict[str, Any]) -> "DevInstance": + return cls( + project_id=project_id, + dir=Path(record.get("dir", "")), + port=int(record.get("port", 0)), + created_at=float(record.get("created_at", 0)), + pid=record.get("pid"), + ) diff --git a/app/living_ui/lifecycle/lifecycle.py b/app/living_ui/lifecycle/lifecycle.py new file mode 100644 index 00000000..8bfc4cb4 --- /dev/null +++ b/app/living_ui/lifecycle/lifecycle.py @@ -0,0 +1,145 @@ +"""AppLifecycle — the facade the actions layer and manager depend on. + +Two operations, one flow for first builds and modifies: + + open_dev(project) boot the DEV environment: the project's current code + on a hidden port with a FRESH schema-only database. + The live app (if any) keeps serving the old code. + promote(project) after a clean walk_verify: deploy the code to the + live environment and destroy the dev copy. + +Composed, never inherited: the provisioner owns dev-env mechanics, the +promoter owns the live boot, and the launch pipeline is injected from the +manager (the same gate/boot pipeline both environments share). +""" + +import secrets +from pathlib import Path +from typing import Any, Awaitable, Callable, Dict + +try: + from loguru import logger +except ImportError: + import logging + + logger = logging.getLogger(__name__) + +from app.living_ui.lifecycle.environment import DevInstance, live_db_exists +from app.living_ui.lifecycle.promoter import Promoter +from app.living_ui.lifecycle.provisioner import DevProvisioner + +LaunchPipeline = Callable[[Path, int, str], Awaitable[Dict[str, Any]]] +LaunchLive = Callable[[str], Awaitable[Dict[str, Any]]] + + +class AppLifecycle: + def __init__( + self, + living_ui_dir: Path, + runner, + launch_pipeline: LaunchPipeline, + launch_live: LaunchLive, + ) -> None: + self.provisioner = DevProvisioner(living_ui_dir, runner) + self.promoter = Promoter(self.provisioner, launch_live) + self._launch_pipeline = launch_pipeline + + # ── dev ──────────────────────────────────────────────────────────────── + async def open_dev(self, project) -> Dict[str, Any]: + """Gate + boot the dev environment for `project` (creating or + refreshing the copy first). The real app is not rebuilt, restarted + or written to. The dev DB is reset on EVERY call: it boots empty and + the migration chain replays, so each iteration re-proves the chain + and starts from the app's true post-migration state. + + Same result envelope as the launch pipeline, plus url/port of the + dev instance and dev=True on success. + """ + from app.factory.host_craftbot import get_factory_host + + if getattr(project, "project_type", "native") == "external": + # Dev envs are pb/-shaped; an external app has no gate or + # migration chain to replay. Changes to externals run live + # (EXTERNAL-APPS-PLAN v1) — callers route them there. + return { + "status": "error", + "step": "dev", + "errors": [ + "External apps have no dev environment — relaunch live " + "via living_ui_notify_ready (changes apply directly)." + ], + } + + host = get_factory_host() + record = host.get_staging_record(project.id) + try: + if ( + record + and Path(record.get("dir", "")).joinpath("manifest.json").exists() + ): + instance = DevInstance.from_record(project.id, record) + self.provisioner.sync_code(project, instance.dir) + self.provisioner.reset_db(instance.dir) + else: + instance = await self.provisioner.create_copy(project) + except Exception as e: + # Never fall back to gating/serving the real project dir — the + # gate's vite build would blank a live app's served UI in place. + return { + "status": "error", + "step": "dev", + "errors": [f"Could not prepare the dev environment: {e}"], + } + + # Reuse (never overwrite) the project's bridge token: a running live + # app carries it in its env, and validate_bridge_token checks the + # current in-memory value — re-minting would cut the live app off + # from the bridge mid-modify. + if not project.bridge_token: + project.bridge_token = secrets.token_urlsafe(32) + + # Record BEFORE booting: a pipeline failure must still leave the + # record in place so living_ui_http redirects there and the next + # open_dev reuses the copy instead of re-cloning. + host.set_staging_record(project.id, instance.to_record()) + + result = await self._launch_pipeline( + instance.dir, instance.port, project.bridge_token + ) + if result["status"] != "success": + return result + + self.provisioner.adopt_process(instance, result.pop("process")) + host.set_staging_record(project.id, instance.to_record()) + + # A change to an app WITH a live database is a modify — re-arm the + # factory machine so it gets the same supervision as a build: fix + # missions on defects, caps, machine announcements. Deterministic + # here, never agent-driven; no-ops when an arc is already in flight. + # An app with no live DB yet is build-era: its machine already owns + # the arc (or is virgin, which stays a first delivery). + if live_db_exists(project.path): + try: + host.begin_modify(project.id) + except Exception as e: + logger.warning(f"[LIVING_UI:DEV] begin_modify failed: {e}") + + logger.info(f"[LIVING_UI:DEV] {project.id} dev env up at {instance.url}") + return { + "status": "success", + "url": instance.url, + "backend_url": instance.url, + "port": instance.port, + "dir": str(instance.dir), + "dev": True, + } + + # ── live ─────────────────────────────────────────────────────────────── + async def promote(self, project) -> Dict[str, Any]: + """Deploy verified code to the live environment (see Promoter).""" + return await self.promoter.promote(project) + + # ── maintenance ──────────────────────────────────────────────────────── + def reap_dev(self, records: Dict[str, Dict[str, Any]]) -> int: + """Startup reaper passthrough (see DevProvisioner.reap_all).""" + return self.provisioner.reap_all(records) diff --git a/app/living_ui/lifecycle/promoter.py b/app/living_ui/lifecycle/promoter.py new file mode 100644 index 00000000..bca74187 --- /dev/null +++ b/app/living_ui/lifecycle/promoter.py @@ -0,0 +1,120 @@ +"""Promoter — the ONLY code path that acts on a LIVE environment. + +promote() deploys a verified change: it boots the real project with the new +code (PocketBase applies new migration files to the real pb_data at boot — +or creates pb_data fresh from the whole chain when this is the app's first +delivery), health-checks it, then destroys the dev environment and every +test record in it. Nothing here ever writes, restores or deletes a live +pb_data — the retired baseline-restore path (finalize_first_delivery) is +exactly the machinery this class replaces. + +before_live_boot hooks run right before the live boot: the reserved slot +for the future pre-promote backup (deferred issue #1 in the plan). A hook +that raises ABORTS the promote — a data-safety hook that silently failed +would be worse than no deploy; the dev env is kept for the retry. +""" + +from typing import Any, Awaitable, Callable, Dict, List + +try: + from loguru import logger +except ImportError: + import logging + + logger = logging.getLogger(__name__) + +from app.living_ui.lifecycle.environment import has_live_env +from app.living_ui.lifecycle.provisioner import DevProvisioner + +LaunchLive = Callable[[str], Awaitable[Dict[str, Any]]] +BeforeLiveBoot = Callable[[Any], None] + + +class Promoter: + def __init__(self, provisioner: DevProvisioner, launch_live: LaunchLive) -> None: + self._provisioner = provisioner + self._launch_live = launch_live + self._before_live_boot: List[BeforeLiveBoot] = [] + + def add_before_live_boot_hook(self, hook: BeforeLiveBoot) -> None: + """Register a hook run with the project right before the live boot + (e.g. a pb_data backup). A raising hook aborts the promote.""" + self._before_live_boot.append(hook) + + async def promote(self, project) -> Dict[str, Any]: + """Deploy the verified code to the live environment. + + Returns the _launch_native result envelope plus `first` (True when + this boot created the app's live database — its first delivery). + On failure the dev environment and its record are KEPT: the live + app is the casualty being repaired, and the next fix iteration + needs the copy. + """ + from app.factory.host_craftbot import get_factory_host + + host = get_factory_host() + is_external = getattr(project, "project_type", "native") == "external" + + # First-vs-update is structural, never a stored flag: does a live + # environment exist before this boot? (For external apps — no pb/ + # shape — that means a promote succeeded before.) + first = not has_live_env(project, host) + + for hook in self._before_live_boot: + try: + hook(project) + except Exception as e: + logger.error(f"[LIVING_UI:PROMOTE] before_live_boot hook failed: {e}") + return { + "status": "error", + "step": "before_live_boot", + "errors": [f"pre-promote hook failed: {e}"], + } + + if is_external: + # External apps run their new code live already (they have no + # dev copy — notify_ready relaunched them in place); promoting + # is pure bookkeeping. + result: Dict[str, Any] = { + "status": "success", + "url": project.url or f"http://127.0.0.1:{project.port}", + "port": project.port, + } + else: + result = await self._launch_live(project.id) + if result.get("status") != "success": + return result + try: + self._provisioner.destroy( + project.id, host.get_staging_record(project.id) + ) + finally: + host.clear_staging_record(project.id) + + result["first"] = first + host.stamp_delivered(project.id) + # Trigger consent (spec TRIGGERS-PLAN): a supervised build or modify + # that delivered is first-party work the user asked for in chat — + # approve its declared triggers. This is also how apps built BEFORE + # the consent feature get approved (observed live 2026-08-06: a + # kanban board gained a user-requested trigger via modify and every + # fire was then consent-blocked, silently). + try: + host.set_triggers_approved(project.id) + except Exception as e: + logger.warning(f"[LIVING_UI:PROMOTE] trigger approval failed: {e}") + + # A tab still showing the pre-promote app must refetch (realtime + # keeps old rows painted through a server restart). + try: + from app.living_ui.broadcast import dispatch_living_ui_data_changed + + dispatch_living_ui_data_changed(project.id) + except Exception: + pass + + logger.info( + f"[LIVING_UI:PROMOTE] {project.id} promoted " + f"({'first delivery' if first else 'update'})" + ) + return result diff --git a/app/living_ui/lifecycle/provisioner.py b/app/living_ui/lifecycle/provisioner.py new file mode 100644 index 00000000..2dfcc575 --- /dev/null +++ b/app/living_ui/lifecycle/provisioner.py @@ -0,0 +1,284 @@ +"""DevProvisioner — creates, refreshes, destroys and reaps DEV environments. + +A dev environment is a full code copy of the project under +living_ui/_staging/project// with its identity rewritten for a hidden +port. Unlike the staging supervisor it replaces, it NEVER clones the live +database: the copy boots with no pb_data at all, PocketBase creates it and +replays the migration chain — so every open_dev is also an implicit +migrations-from-empty test, and no real user data ever enters an +environment the agent or verifier writes to. + +Composition mirrors LivingUIRunner: the lifecycle constructs and drives +this class; it never reaches back into the manager or the registry. The +authoritative "a dev copy exists" record lives in the factory host sidecar +(.factory/host.json, key "staging" — historical name, kept so records and +reapers from older versions stay compatible). +""" + +import json +import os +import re +import shutil +import signal +import socket +import subprocess +import time +from pathlib import Path +from typing import Any, Dict, Optional + +try: + from loguru import logger +except ImportError: + import logging + + logger = logging.getLogger(__name__) + +from app.living_ui.lifecycle.environment import DevInstance + +# Outside the manager's 3100-3199 pool on purpose: _load_projects rebuilds +# port bookkeeping from registered projects only, and cleanup_on_startup's +# orphan killer scans that range — dev envs own their ports and their reaping. +DEV_PORT_RANGE = (3900, 3999) + +# Same guard the wizard uses for its ids: nothing outside this pattern ever +# becomes part of an rmtree'd path. +_ID_RE = re.compile(r"^[A-Za-z0-9_-]{4,64}$") + +# What a dev copy takes from the real project. pb_data is deliberately +# ABSENT (fresh DB from migration replay at boot); pb_public too — the +# gate's build step recreates it inside the copy. triggers.json MUST +# travel: without it the copy's trigger guard declares nothing, every ⚡ +# fire 400s, the walker fails an unfixable "defect", and the arc sticks +# (observed live 2026-08-06, kanban board). +_COPY_FILES = ("manifest.json", "operations.json", "triggers.json", "LIVING_UI.md") +_COPY_CREDS = (".superuser", ".agent-token") +_COPY_DIRS = ("frontend", "pb/pb_hooks", "pb/pb_migrations", ".lui", "reference") + +# What sync_code refreshes on each fix-mission iteration: the agent-owned +# paths (ownership rule, agent-guide §1) — never manifest.json (the copy's +# port rewrite must survive). +_SYNC_FILES = ("operations.json", "triggers.json", "LIVING_UI.md") +_SYNC_DIRS = ("frontend/src", "pb/pb_hooks", "pb/pb_migrations", "reference") +_SYNC_PKG = ("frontend/package.json", "frontend/package-lock.json") + + +class DevProvisioner: + """Creates, refreshes, destroys and reaps dev environments. Knows nothing + about the manager's registry, sessions or broadcasting — the lifecycle + composes this class; it never reaches back.""" + + def __init__(self, living_ui_dir: Path, runner) -> None: + self.living_ui_dir = Path(living_ui_dir) + self.root = self.living_ui_dir / "_staging" / "project" + self.runner = runner + # Live process handles, keyed by project id. Best-effort only — + # after a CraftBot restart the pid in the sidecar record is all + # that's left, and destroy/reap fall back to it. + self._processes: Dict[str, subprocess.Popen] = {} + + # ── create / refresh ─────────────────────────────────────────────────── + async def create_copy(self, project) -> DevInstance: + """Build a fresh dev copy of `project` (code only — no database) and + rewrite its identity for a hidden port. Does NOT boot it — the + lifecycle runs the shared launch pipeline against the returned dir. + Raises on failure; a partial copy is removed.""" + if not _ID_RE.match(project.id or ""): + raise ValueError(f"unsafe project id for dev copy: {project.id!r}") + src = Path(project.path) + if not (src / "manifest.json").exists(): + raise FileNotFoundError(f"not a Living UI project: {src}") + + dev_dir = self.root / project.id + if dev_dir.exists(): + self._guarded_rmtree(dev_dir) + dev_dir.mkdir(parents=True) + + try: + for rel in _COPY_FILES + _COPY_CREDS: + f = src / rel + if f.exists(): + shutil.copy2(f, dev_dir / rel) + for rel in _COPY_DIRS: + d = src / rel + if d.is_dir(): + # node_modules rides along inside frontend/ — without it + # the gate cold-installs for up to 600 s per dev boot. + shutil.copytree(d, dev_dir / rel, symlinks=True) + (dev_dir / "logs").mkdir(exist_ok=True) + + port = self._free_port() + self._rewrite_manifest(dev_dir, port) + + # The manifest rewrite invalidated the system-hash canon; + # kit-sync re-vendors the kit and re-records hashes (same + # recovery the ZIP-import path uses) — without it the gate's + # ownership step fails with "modified: manifest.json". + await self.runner.kit_sync(dev_dir) + except Exception: + self._guarded_rmtree(dev_dir) + raise + + instance = DevInstance( + project_id=project.id, + dir=dev_dir, + port=port, + created_at=time.time(), + ) + logger.info( + f"[LIVING_UI:DEV] created dev copy of {project.id} at " + f"{dev_dir} (port {port})" + ) + return instance + + def sync_code(self, project, dev_dir: Path) -> None: + """Refresh the agent-owned paths real → dev (fix-mission iterations + edit the real files; the dev copy is what gets gated and served). + Keeps the rewritten manifest.""" + src = Path(project.path) + dev_dir = Path(dev_dir) + if not (dev_dir / "manifest.json").exists(): + raise FileNotFoundError(f"dev copy missing at {dev_dir}") + + # A changed package.json means new/changed deps: drop node_modules so + # the pipeline's install step runs for real instead of being skipped. + for rel in _SYNC_PKG: + s, d = src / rel, dev_dir / rel + if s.exists() and (not d.exists() or s.read_bytes() != d.read_bytes()): + shutil.copy2(s, d) + nm = dev_dir / "frontend" / "node_modules" + if nm.is_dir(): + logger.info( + "[LIVING_UI:DEV] package.json changed — " + "clearing dev node_modules for a fresh install" + ) + self._guarded_rmtree(nm) + + for rel in _SYNC_FILES: + s = src / rel + if s.exists(): + shutil.copy2(s, dev_dir / rel) + for rel in _SYNC_DIRS: + s, d = src / rel, dev_dir / rel + if s.is_dir(): + if d.exists(): + self._guarded_rmtree(d) + shutil.copytree(s, d, symlinks=True) + + def reset_db(self, dev_dir: Path) -> None: + """Drop the dev copy's database so the next boot recreates it from + the migration chain. Called on every open_dev reuse: each iteration + re-proves the chain replays cleanly from empty, and stale test + records never accumulate into false verifier context.""" + pb_data = Path(dev_dir) / "pb" / "pb_data" + if pb_data.exists(): + self._guarded_rmtree(pb_data) + + # ── process bookkeeping ──────────────────────────────────────────────── + def adopt_process(self, instance: DevInstance, process) -> None: + instance.process = process + instance.pid = process.pid + self._processes[instance.project_id] = process + + # ── destroy / reap ───────────────────────────────────────────────────── + def destroy(self, project_id: str, record: Optional[Dict[str, Any]]) -> None: + """Kill the dev process and delete the copy. Idempotent and + best-effort: a half-dead dev env must never block a promote.""" + process = self._processes.pop(project_id, None) + if process is not None and process.poll() is None: + self._kill(process=process) + elif record and record.get("pid"): + self._kill(pid=int(record["pid"])) + + dev_dir = ( + Path(record["dir"]) + if record and record.get("dir") + else (self.root / project_id) + ) + if dev_dir.exists(): + try: + self._guarded_rmtree(dev_dir) + logger.info(f"[LIVING_UI:DEV] destroyed dev copy of {project_id}") + except Exception as e: + logger.warning(f"[LIVING_UI:DEV] failed to delete {dev_dir}: {e}") + + def reap_all(self, records: Dict[str, Dict[str, Any]]) -> int: + """Startup reaper: no dev copy is legitimately alive when CraftBot + boots (their missions died with the process), so kill every recorded + pid and delete everything under the dev root — including dirs with + no surviving record. Deliberate, unlike the blind orphan rmtree in + cleanup_on_startup (which skips _staging entirely).""" + reaped = 0 + for project_id, record in records.items(): + self.destroy(project_id, record) + reaped += 1 + if self.root.exists(): + for leftover in self.root.iterdir(): + try: + self._guarded_rmtree(leftover) + reaped += 1 + logger.info(f"[LIVING_UI:DEV] reaped leftover {leftover.name}") + except Exception as e: + logger.warning(f"[LIVING_UI:DEV] failed to reap {leftover}: {e}") + return reaped + + # ── internals ────────────────────────────────────────────────────────── + def _guarded_rmtree(self, target: Path) -> None: + """Only ever delete inside living_ui/_staging/ — the same + strict-ancestor discipline delete_project adopted after rmtree wiped + the working tree twice (2026-07-25/26). By construction this also + makes it impossible for the provisioner to delete a LIVE pb_data: + live projects do not live under the dev root.""" + resolved = Path(target).resolve() + dev_root = (self.living_ui_dir / "_staging").resolve() + if dev_root not in resolved.parents: + raise ValueError(f"refusing to delete {resolved} — outside {dev_root}") + shutil.rmtree(resolved) + + def _free_port(self) -> int: + for port in range(DEV_PORT_RANGE[0], DEV_PORT_RANGE[1] + 1): + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("127.0.0.1", port)) + return port + except OSError: + continue + raise RuntimeError("No free port in the dev range 3900-3999") + + def _rewrite_manifest(self, dev_dir: Path, port: int) -> None: + """Rewrite the copy's identity: its port (`lui ops/run/data` derive + their base URL from manifest.port — a stale port would make CLI + calls from the dev dir hit the LIVE app) and `env: "dev"`, which the + A2APP identity endpoint surfaces so any client can structurally + confirm which environment a port belongs to.""" + manifest_path = dev_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + old_port = manifest.get("port") + manifest["port"] = port + manifest["env"] = "dev" + if isinstance(manifest.get("pipeline"), dict) and old_port: + manifest["pipeline"] = json.loads( + json.dumps(manifest["pipeline"]).replace(str(old_port), str(port)) + ) + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + def _kill(self, process=None, pid: Optional[int] = None) -> None: + try: + if process is not None: + process.terminate() + try: + process.wait(timeout=5) + except Exception: + process.kill() + elif pid: + os.kill(pid, signal.SIGTERM) + time.sleep(0.5) + try: + os.kill(pid, 0) + except OSError: + return # already gone + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + except Exception as e: + logger.warning(f"[LIVING_UI:DEV] kill failed: {e}") diff --git a/app/living_ui/manager.py b/app/living_ui/manager.py index 7b96ddf8..d32cb1f2 100644 --- a/app/living_ui/manager.py +++ b/app/living_ui/manager.py @@ -62,6 +62,12 @@ class LivingUIProject: session_id: Optional[str] = None auto_launch: bool = False # Auto-launch on CraftBot startup log_cleanup: bool = True # Clean logs on restart + # Backups of live pb_data (spec docs/plans/living-ui-backups-plan.md). + # Default ON (D1): the user who never opens settings is the one who + # needs a backup. No-ops until a live DB exists; external apps N/A. + backups_enabled: bool = True + backup_interval: str = "daily" # hourly | 6h | daily | weekly + backup_keep: int = 7 # scheduled-pool retention (1-30) style_pack: str = "" # wizard-chosen default style pack (host may override) # Display icon: "lucide:" (picker) or "file:" (uploaded, # doubles as the app's favicon). @@ -100,6 +106,9 @@ def to_dict(self) -> Dict[str, Any]: "sessionId": self.session_id, "autoLaunch": self.auto_launch, "logCleanup": self.log_cleanup, + "backupsEnabled": self.backups_enabled, + "backupInterval": self.backup_interval, + "backupKeep": self.backup_keep, "stylePack": self.style_pack, "icon": self.icon, "uiTheme": self.ui_theme, @@ -147,11 +156,36 @@ def __init__(self, workspace_root: Path): self.runner = LivingUIRunner(Path(PROJECT_ROOT) / "living-ui") - # Staging copies of delivered apps (modify-era data safety). Composed - # like runner: the supervisor never reaches back into the manager. - from app.living_ui.staging import StagingSupervisor + # Unified dev/live lifecycle: every code change (first build or + # modify) develops and verifies in a DEV environment (code copy + + # fresh schema-only DB on a hidden port); a clean verify PROMOTES it + # to live. Composed like runner: the lifecycle never reaches back + # into the manager beyond the two callables injected here. + from app.living_ui.lifecycle import AppLifecycle + + self.lifecycle = AppLifecycle( + self.living_ui_dir, + self.runner, + self._run_launch_pipeline, + self.launch_and_verify, + ) - self.staging = StagingSupervisor(self.living_ui_dir, self.runner) + # Backups of live pb_data (spec docs/plans/living-ui-backups-plan.md). + # Composed like the lifecycle: the service never reaches back. The + # watchdog drives the schedule; ONE lock serializes captures; the + # in-flight set keeps the scheduler out of promotes/restores (and + # vice versa). + from app.living_ui.lifecycle import BackupService + + self.backups = BackupService(self.living_ui_dir) + self._backup_lock = asyncio.Lock() + self._live_ops: set = set() # project ids mid-promote/mid-restore + self._backups_inflight: set = set() # ids with a capture task queued/running + # Pre-promote backup (lifecycle plan deferred issue #1): snapshot the + # live pb_data right before every promote boot over existing data. + # Sync hook by contract; a raising capture ABORTS the promote — never + # deploy over data we just failed to protect. + self.lifecycle.promoter.add_before_live_boot_hook(self._pre_promote_backup) # Load existing projects self._load_projects() @@ -284,6 +318,17 @@ async def _watchdog_loop(self) -> None: await asyncio.sleep(self.WATCHDOG_INTERVAL) for project_id, project in list(self.projects.items()): + # Backups are due-checked for EVERY project, before the + # running gate — a stopped app with a live DB still backs + # up (via the stopped capture path). + try: + self._maybe_schedule_backup(project) + except Exception as e: + logger.warning( + f"[LIVING_UI:BACKUP] schedule check failed for " + f"{project_id}: {e}" + ) + if project.status != "running": # Clear retry count if project is no longer running retry_counts.pop(project_id, None) @@ -383,6 +428,134 @@ async def _watchdog_loop(self) -> None: logger.error(f"[LIVING_UI:WATCHDOG] Unexpected error: {e}") await asyncio.sleep(self.WATCHDOG_INTERVAL) + # ======================================================================== + # Backups (spec docs/plans/living-ui-backups-plan.md) + # ======================================================================== + + _BACKUP_INTERVALS = { + "hourly": 3600, + "6h": 6 * 3600, + "daily": 86400, + "weekly": 7 * 86400, + } + + def _maybe_schedule_backup(self, project) -> None: + """Watchdog tick: start a due scheduled backup as a background task. + Sync and cheap — one sidecar read past the structural gates.""" + from app.factory.host_craftbot import get_factory_host + from app.living_ui.lifecycle import live_db_exists + + if ( + not project.backups_enabled + or getattr(project, "project_type", "native") == "external" + or project.id in self._live_ops + or project.id in self._backups_inflight + or not live_db_exists(project.path) + ): + return + state = get_factory_host().backup_state(project.id) + interval = self._BACKUP_INTERVALS.get(project.backup_interval, 86400) + # Absent last_at -> due now: first-enable AND catch-up after a + # restart/overdue sleep both fall out of the same rule. + if state["last_at"] is not None and time.time() - state["last_at"] < interval: + return + self._backups_inflight.add(project.id) + asyncio.create_task(self._run_scheduled_backup(project)) + + async def _run_scheduled_backup(self, project) -> None: + """One scheduled capture + prune + sidecar record. Failure never + touches the app (FR10): log, record, retry at the next due tick.""" + from app.factory.host_craftbot import get_factory_host + + host = get_factory_host() + try: + async with self._backup_lock: # serialize captures globally (NFR) + if project.id in self._live_ops: + return # promote/restore began while queued — next tick + entry = await self._capture_auto(project, "scheduled") + self.backups.store.prune(project.id, "scheduled", project.backup_keep) + host.record_backup_ok(project.id, entry.ts) + except Exception as e: + logger.warning( + f"[LIVING_UI:BACKUP] scheduled backup failed for {project.id}: {e}" + ) + try: + host.record_backup_error(project.id, str(e)) + except Exception: + pass + finally: + self._backups_inflight.discard(project.id) + + async def _capture_auto(self, project, trigger: str): + """Running app → PB's atomic backup API; stopped → snapshot path + (off-loop — sqlite backup + zip can take seconds).""" + if project.status == "running" and project.port: + return await self.backups.capture_running(project, trigger) + return await asyncio.to_thread(self.backups.capture_stopped, project, trigger) + + async def backup_now(self, project_id: str) -> dict: + """User-driven manual backup (FR8). Manual-pool: never auto-pruned.""" + from app.factory.host_craftbot import get_factory_host + from app.living_ui.lifecycle import live_db_exists + + project = self.projects.get(project_id) + if not project: + return {"status": "error", "errors": [f"Unknown project: {project_id}"]} + if getattr(project, "project_type", "native") == "external": + return {"status": "error", "errors": ["External apps have no pb_data."]} + if not live_db_exists(project.path): + return { + "status": "error", + "errors": ["No live database yet — nothing to back up."], + } + if project_id in self._live_ops: + return { + "status": "error", + "errors": ["A promote/restore is in flight — retry shortly."], + } + host = get_factory_host() + try: + async with self._backup_lock: + entry = await self._capture_auto(project, "manual") + host.record_backup_ok(project_id, entry.ts) + return { + "status": "success", + "filename": entry.filename, + "size": entry.size, + } + except Exception as e: + logger.warning( + f"[LIVING_UI:BACKUP] manual backup failed for {project_id}: {e}" + ) + try: + host.record_backup_error(project_id, str(e)) + except Exception: + pass + return {"status": "error", "errors": [str(e)]} + + def _pre_promote_backup(self, project) -> None: + """before_live_boot hook (lifecycle deferred issue #1): snapshot live + pb_data right before the promote boot. First deliveries (no live DB) + and externals (no pb/) no-op. RAISES on failure — the promoter + aborts, by contract: never deploy over data we failed to protect.""" + from app.factory.host_craftbot import get_factory_host + from app.living_ui.lifecycle import live_db_exists + from app.living_ui.lifecycle.backups import PRE_PROMOTE_KEEP + + if getattr(project, "project_type", "native") == "external": + return + if not live_db_exists(project.path): + return + entry = self.backups.capture_stopped(project, "pre_promote") + self.backups.store.prune(project.id, "pre_promote", PRE_PROMOTE_KEEP) + try: + # A fresh capture is a fresh capture: reset the scheduled clock + # so promote-heavy days don't also stack near-identical + # scheduled archives minutes later. + get_factory_host().record_backup_ok(project.id, entry.ts) + except Exception: + pass + async def _escalate_crash(self, project_id: str, crash_targets: List[str]) -> None: """ Escalate a crash to the agent by creating a fix task. @@ -550,6 +723,9 @@ def _load_projects(self) -> None: session_id=project_data.get("sessionId"), auto_launch=project_data.get("autoLaunch", False), log_cleanup=project_data.get("logCleanup", True), + backups_enabled=project_data.get("backupsEnabled", True), + backup_interval=project_data.get("backupInterval", "daily"), + backup_keep=project_data.get("backupKeep", 7), style_pack=project_data.get("stylePack", ""), icon=project_data.get("icon"), ui_theme=project_data.get("uiTheme"), @@ -803,10 +979,10 @@ async def _run_launch_pipeline( install → validation gate → serve → health → hook-load scan → smoke. Registry-free on purpose: `_launch_native` runs it on the real project - and adds status/persistence around it; `launch_staging` runs the SAME - pipeline on a staging copy — one definition means fix missions get - identical evidence quality (boot-log excerpts, hook-load failures) - in both eras. + and adds status/persistence around it; the lifecycle's `open_dev` + runs the SAME pipeline on a dev copy — one definition means fix + missions get identical evidence quality (boot-log excerpts, + hook-load failures) in both environments. Returns {"status": "success", "process": Popen} — caller owns the process — or {"status": "error", "step": ..., "errors": [...]}. @@ -1058,7 +1234,7 @@ def _log_since_boot(limit_lines: int = 30) -> str: async def _launch_native(self, project: LivingUIProject) -> dict: """Native launch of the REAL project: the shared pipeline plus registry - state (status, url, persistence) and the pristine-baseline hook. + state (status, url, persistence). One PocketBase process serves both the API and the built frontend (living-ui spec D5); errors come back machine-readable so the @@ -1100,29 +1276,6 @@ async def _launch_native(self, project: LivingUIProject) -> dict: project.error = None self._save_projects() - # Pristine-baseline snapshot: taken ONCE, at the first successful - # launch of a never-delivered app — before the agent or verifier has - # created any test records. finalize_first_delivery() restores it - # right before the delivery announce so the user's first sight of the - # app is junk-free. Best-effort by design: a failed snapshot must - # never block a launch (worst case the app delivers with test data, - # which is today's behavior). - try: - from app.factory.host_craftbot import get_factory_host - from app.living_ui.pb_data_io import snapshot_pb_data - - baseline = project_path / ".snapshots" / "baseline" - if ( - getattr(project, "project_type", "native") != "external" - and not baseline.exists() - and not get_factory_host().is_delivered(project.id) - ): - snapshot_pb_data( - project_path / "pb" / "pb_data", baseline, self.living_ui_dir - ) - except Exception as e: - logger.warning(f"[LIVING_UI] baseline snapshot skipped: {e}") - logger.info(f"[LIVING_UI] {project.name} running at {project.url}") return { "status": "success", @@ -1131,231 +1284,172 @@ async def _launch_native(self, project: LivingUIProject) -> dict: "port": project.port, } - async def finalize_first_delivery(self, project_id: str) -> dict: - """Restore the pristine pb_data baseline and relaunch, so the app the - user is about to be handed contains no agent/verifier test records. - - Called from walk_verify's clean branch, on a never-delivered app, - AFTER the verifier passed against the live (junk-filled) DB and - BEFORE the delivery announce. Migration files written during the - build re-apply on the restored DB at boot (they are absent from its - _migrations table), so the delivered schema is current — the gate - proves the full migration chain replays cleanly on every validate. - - A missing baseline (legacy project, snapshot failure at first launch) - is NOT an error: we skip the restore and deliver as today, never - guess-wipe. Returns {"status": "success"} or an error dict in the - _launch_native envelope. - """ + async def open_dev(self, project_id: str) -> dict: + """Boot the DEV environment for a code change (first build or + modify): the project's current code on a hidden port with a fresh + schema-only DB. See lifecycle.AppLifecycle.open_dev.""" project = self.projects.get(project_id) if not project: return { "status": "error", - "step": "finalize", + "step": "dev", "errors": [f"Unknown project: {project_id}"], } - project_path = Path(project.path) - baseline = project_path / ".snapshots" / "baseline" - if not (baseline / "data.db").exists(): - logger.info( - f"[LIVING_UI] no baseline for {project_id} — delivering without restore" - ) - return {"status": "success", "restored": False} + return await self.lifecycle.open_dev(project) - from app.living_ui.pb_data_io import restore_pb_data - - # Stop the server before touching pb_data (a live writer during the - # restore corrupts both sides). Don't flip status mid-sequence — the - # watchdog restarts anything still marked "running" with a dead port, - # and a half-finalized app must not be relaunched under our feet. - project.status = "stopped" - if project.process: - self._terminate_process(project.process) - project.process = None - if project.port and self._is_port_in_use(project.port): - self._kill_process_on_port(project.port) - - try: - restore_pb_data( - baseline, project_path / "pb" / "pb_data", self.living_ui_dir - ) - except Exception as e: - # pb_data may now be gone/partial — a plain start would boot an - # empty DB. Fall through to the full pipeline, which re-applies - # migrations and re-verifies before anyone is told "ready". - logger.error(f"[LIVING_UI] baseline restore failed: {e}") - return await self._launch_native(project) - - try: - project.process = await self.runner.start( - project_path, project.port, bridge_token=project.bridge_token - ) - if not await self.runner.wait_healthy(project.port): - raise RuntimeError(f"/api/health not responding on :{project.port}") - except Exception as e: - logger.warning( - f"[LIVING_UI] slim relaunch after restore failed ({e}) — " - "falling back to the full pipeline" - ) - return await self._launch_native(project) - - project.status = "running" - project.url = f"http://127.0.0.1:{project.port}" - project.backend_url = project.url - project.error = None - self._save_projects() - # The user's tab may still render the verifier's test records from - # before the restore (realtime keeps old rows painted through a - # server restart) — tell the frontend to refetch so the first thing - # the user sees is the pristine state. - try: - from app.living_ui.broadcast import dispatch_living_ui_data_changed - - dispatch_living_ui_data_changed(project_id) - except Exception: - pass - # Trigger consent: a supervised build that delivered is first-party — - # approve its declared triggers (mirror of finalize_modify's grant). + async def promote(self, project_id: str) -> dict: + """Deploy verified code to the live environment and destroy the dev + copy. See lifecycle.Promoter.promote.""" + project = self.projects.get(project_id) + if not project: + return { + "status": "error", + "step": "promote", + "errors": [f"Unknown project: {project_id}"], + } + # Visible to the backup scheduler: no scheduled capture may start + # mid-promote (the pre-promote hook is the sanctioned one). + self._live_ops.add(project_id) try: - from app.factory.host_craftbot import get_factory_host - - get_factory_host().set_triggers_approved(project_id) - except Exception as e: - logger.warning(f"[LIVING_UI] trigger approval on delivery failed: {e}") - logger.info(f"[LIVING_UI] {project_id} finalized for first delivery") - return {"status": "success", "restored": True} - - async def launch_staging(self, project_id: str) -> dict: - """Gate + boot the STAGING copy of a delivered app (creating or - refreshing it first). The real app is not rebuilt, restarted or - written to — it keeps serving the old working code while the change - is developed and verified in the copy. - - Same result envelope as _launch_native, plus url/port of the staging - instance on success. + return await self.lifecycle.promote(project) + finally: + self._live_ops.discard(project_id) + + async def restore_backup(self, project_id: str, filename: str) -> dict: + """User-initiated restore of a pb_data backup (FR9) — the SECOND + sanctioned live-write path (the first is migration replay during + promote; see lifecycle/__init__). Made reversible rather than + friction-guarded: the current live state is captured first, so a + wrong restore is undone by restoring THAT archive. + + stop → pre-restore capture (abort if it fails: never destroy state + we failed to save) → replace pb_data → full-pipeline relaunch + (migrations newer than the archive re-apply at boot) → refetch + broadcast. Never agent-invocable — settings surface only. """ - from app.factory.host_craftbot import get_factory_host - from app.living_ui.staging import StagingInstance + from app.living_ui.pb_data_io import restore_pb_data project = self.projects.get(project_id) if not project: return { "status": "error", - "step": "staging", + "step": "restore", "errors": [f"Unknown project: {project_id}"], } if getattr(project, "project_type", "native") == "external": - # Staging is pb/-shaped; an external app has no clonable DB or - # gate. Changes to externals run live (EXTERNAL-APPS-PLAN v1). return { "status": "error", - "step": "staging", - "errors": [ - "External apps have no staging mode — relaunch live via " - "living_ui_notify_ready (changes apply directly)." - ], + "step": "restore", + "errors": ["External apps have no pb_data backups."], } - - host = get_factory_host() - record = host.get_staging_record(project_id) - try: - if ( - record - and Path(record.get("dir", "")).joinpath("manifest.json").exists() - ): - instance = StagingInstance.from_record(project_id, record) - self.staging.sync_code(project, instance.dir) - else: - instance = await self.staging.create_copy(project) - except Exception as e: - # Never fall back to gating/serving the real project dir — that - # is exactly the live-UI blanking this mode exists to prevent. + entry = next( + ( + e + for e in self.backups.store.list_backups(project_id) + if e.filename == filename + ), + None, + ) + if entry is None: return { "status": "error", - "step": "staging", - "errors": [f"Could not prepare the staging copy: {e}"], + "step": "restore", + "errors": [f"No such backup: {filename}"], + } + if project_id in self._live_ops: + return { + "status": "error", + "step": "restore", + "errors": ["Another promote/restore is in flight — retry shortly."], } - # Reuse (never overwrite) the project's bridge token: the live app's - # running process carries it in its env, and validate_bridge_token - # checks the current in-memory value — re-minting would cut the live - # app off from the bridge mid-modify. - if not project.bridge_token: - project.bridge_token = secrets.token_urlsafe(32) - - # Record BEFORE booting: a pipeline failure must still leave the - # record in place so living_ui_http redirects there and the next - # notify_ready reuses the copy instead of re-cloning. - host.set_staging_record(project_id, instance.to_record()) - - result = await self._run_launch_pipeline( - instance.dir, instance.port, project.bridge_token - ) - if result["status"] != "success": - return result - - self.staging.adopt_process(instance, result.pop("process")) - host.set_staging_record(project_id, instance.to_record()) - - # A modify is now demonstrably in progress (staging is up) — re-arm - # the factory machine so the modify gets the same supervision as a - # build: fix missions on defects, caps, machine announcements - # (LIFECYCLE-PLAN Phase 2). Deterministic here, never agent-driven; - # no-ops when a modify/fix arc is already in flight. + self._live_ops.add(project_id) try: - host.begin_modify(project_id) - except Exception as e: - logger.warning(f"[LIVING_UI:STAGING] begin_modify failed: {e}") + was_running = project.status == "running" + await self.stop_project(project_id) - logger.info(f"[LIVING_UI:STAGING] {project_id} staging up at {instance.url}") - return { - "status": "success", - "url": instance.url, - "backend_url": instance.url, - "port": instance.port, - "staging": True, - } + # FR9 2a — the abort-on-failure safety net. + try: + pre = await asyncio.to_thread( + self.backups.capture_stopped, project, "manual" + ) + try: + from app.factory.host_craftbot import get_factory_host + + get_factory_host().record_backup_ok(project_id, pre.ts) + except Exception: + pass + except Exception as e: + result = await self.launch_and_verify(project_id) if was_running else {} + return { + "status": "error", + "step": "pre_restore_backup", + "errors": [ + f"Could not back up the CURRENT state ({e}) — restore " + "aborted, nothing was changed." + + ( + "" + if result.get("status") in ("success", None) + else " Relaunch of the untouched app also failed." + ) + ], + } - async def finalize_modify(self, project_id: str) -> dict: - """The flip, after a clean staging verify: relaunch the REAL project - (the gate rebuilds its pb_public; new migration files apply to the - real pb_data at boot — user data stays in place), then destroy the - staging copy and every test record with it. + restore_error = None + try: + snapshot = await asyncio.to_thread(self.backups.prepare_restore, entry) + await asyncio.to_thread( + restore_pb_data, + snapshot, + Path(project.path) / "pb" / "pb_data", + self.living_ui_dir, + ) + except Exception as e: + restore_error = str(e) + finally: + self.backups.cleanup_restore(entry) - On failure the staging copy and its record are KEPT — the real app - is the casualty being repaired, and the next fix iteration needs the - copy. - """ - from app.factory.host_craftbot import get_factory_host + # Relaunch through the full pipeline either way: on success the + # restored DB boots (newer migrations re-apply); on failure + # pb_data may be partial and the gate/boot is the honest probe. + result = await self.launch_and_verify(project_id) + if restore_error is not None: + return { + "status": "error", + "step": "restore", + "errors": [ + f"Restore failed: {restore_error}. A backup of the " + f"pre-restore state was kept ({pre.filename}).", + *result.get("errors", [])[:5], + ], + } + if result.get("status") != "success": + return { + "status": "error", + "step": "relaunch", + "errors": [ + "pb_data was restored but the app failed to relaunch. " + f"Pre-restore state is kept as {pre.filename}.", + *result.get("errors", [])[:5], + ], + } - result = await self.launch_and_verify(project_id) - if result["status"] != "success": - return result + # Open tabs still paint pre-restore rows through the restart. + try: + from app.living_ui.broadcast import dispatch_living_ui_data_changed - host = get_factory_host() - try: - self.staging.destroy(project_id, host.get_staging_record(project_id)) + dispatch_living_ui_data_changed(project_id) + except Exception: + pass + logger.info(f"[LIVING_UI:BACKUP] {project_id} restored from {filename}") + return { + "status": "success", + "restored": filename, + "pre_restore_backup": pre.filename, + "url": result.get("url"), + } finally: - host.clear_staging_record(project_id) - # Trigger consent (spec TRIGGERS-PLAN): a supervised modify that - # delivered is first-party work the user asked for in chat — approve - # its declared triggers. This is also how apps built BEFORE the - # consent feature get approved (observed live 2026-08-06: a kanban - # board gained a user-requested trigger via modify and every fire - # was then consent-blocked, silently). - try: - host.set_triggers_approved(project_id) - except Exception as e: - logger.warning(f"[LIVING_UI] trigger approval on flip failed: {e}") - # A tab still showing the pre-flip app must refetch (same stale-view - # hazard as finalize_first_delivery's baseline restore). - try: - from app.living_ui.broadcast import dispatch_living_ui_data_changed - - dispatch_living_ui_data_changed(project_id) - except Exception: - pass - return result + self._live_ops.discard(project_id) async def launch_and_verify(self, project_id: str) -> dict: """ @@ -1728,15 +1822,17 @@ def cleanup_on_startup(self) -> None: if killed_count > 0: logger.info(f"[LIVING_UI] Killed {killed_count} orphan process(es)") - # 2. Clean up orphan project folders - orphan_count = self._cleanup_orphan_folders() + # 2. Log orphan project folders (do NOT delete — deleting them at boot + # has destroyed real user projects; logging is the safe behavior). + orphan_count = self._log_orphan_folders() if orphan_count > 0: - logger.info(f"[LIVING_UI] Removed {orphan_count} orphan folder(s)") + logger.info(f"[LIVING_UI] Found {orphan_count} orphan folder(s) (left in place)") - # 2b. Reap staging copies. None is legitimately alive at boot (their - # modify missions died with the previous process), but their - # PocketBase instances outlive us — kill by recorded pid, delete the - # copies, clear the records so nothing redirects to a dead port. + # 2b. Reap dev environments. None is legitimately alive at boot + # (their build/modify missions died with the previous process), but + # their PocketBase instances outlive us — kill by recorded pid, + # delete the copies, clear the records so nothing redirects to a + # dead port. try: from app.factory.host_craftbot import get_factory_host @@ -1746,13 +1842,13 @@ def cleanup_on_startup(self) -> None: record = host.get_staging_record(pid_) if record: records[pid_] = record - reaped = self.staging.reap_all(records) + reaped = self.lifecycle.reap_dev(records) for pid_ in records: host.clear_staging_record(pid_) if reaped: - logger.info(f"[LIVING_UI] Reaped {reaped} staging leftover(s)") + logger.info(f"[LIVING_UI] Reaped {reaped} dev-env leftover(s)") except Exception as e: - logger.warning(f"[LIVING_UI] staging reap failed: {e}") + logger.warning(f"[LIVING_UI] dev-env reap failed: {e}") # 3. Reset all project statuses to 'stopped' and clear process references for project in self.projects.values(): @@ -1765,12 +1861,16 @@ def cleanup_on_startup(self) -> None: logger.info("[LIVING_UI] Startup cleanup complete") - def _cleanup_orphan_folders(self) -> int: + def _log_orphan_folders(self) -> int: """ - Delete project folders that are not tracked in the registry. + Log project folders that are not tracked in the registry. + + Orphan folders are deliberately NOT deleted: deleting them at boot has + destroyed real user projects. We only surface them so they can be + recovered or removed manually. Returns: - Number of orphan folders deleted + Number of orphan folders found """ if not self.living_ui_dir.exists(): return 0 @@ -1778,25 +1878,22 @@ def _cleanup_orphan_folders(self) -> int: tracked_paths = {Path(p.path) for p in self.projects.values()} orphan_count = 0 - # _staging is workspace infrastructure, not an orphan project: the - # wizard stages reference files under it (with its own age-based - # sweeper) and StagingSupervisor keeps modify-era app copies there - # (reaped deliberately — kill recorded pid, then delete — by - # reap_orphans(), not by this blind rmtree). - skip_names = {"_staging"} + # _staging and _backups are workspace infrastructure, not orphan + # projects: the wizard stages reference files under _staging (with + # its own age-based sweeper) and DevProvisioner keeps dev-env app + # copies there. _backups holds pb_data archives that must OUTLIVE + # their project. Skip both so they never show up as orphans. + skip_names = {"_staging", "_backups"} for folder in self.living_ui_dir.iterdir(): if folder.name in skip_names: continue if folder.is_dir() and folder not in tracked_paths: - try: - shutil.rmtree(folder) - logger.info(f"[LIVING_UI] Deleted orphan folder: {folder.name}") - orphan_count += 1 - except Exception as e: - logger.warning( - f"[LIVING_UI] Failed to delete orphan folder {folder}: {e}" - ) + logger.warning( + f"[LIVING_UI] Orphan folder (not tracked in registry, left " + f"in place): {folder.name}" + ) + orphan_count += 1 return orphan_count @@ -1880,9 +1977,10 @@ async def create_project( def _register_acquired(self, project: LivingUIProject, *, delivered: bool) -> None: """Every entry point (scaffold / marketplace / import) lands here after its starting state is on disk (LIFECYCLE-PLAN Phase 3): - registry + persistence + session, and — for sources that arrive as - finished apps — the delivered flag that keys every later data-safety - mode (staging verifies, no baseline restore).""" + registry + persistence + session. `delivered` means the app ARRIVED + finished (marketplace/import): its delivery timestamp is stamped and + trigger consent stays fail-closed. Data safety no longer keys on it + — that's structural (lifecycle.live_db_exists).""" # Provenance: which CraftBot acquired this project (the manifest's # craftbotVersion separately records the original creator's version). if not project.craftbot_version: @@ -1904,10 +2002,10 @@ def _register_acquired(self, project: LivingUIProject, *, delivered: bool) -> No try: from app.factory.host_craftbot import get_factory_host - get_factory_host().mark_delivered(project.id) + get_factory_host().stamp_delivered(project.id) except Exception as e: logger.warning( - f"[LIVING_UI] mark_delivered failed for {project.id}: {e}" + f"[LIVING_UI] stamp_delivered failed for {project.id}: {e}" ) else: # Trigger-plane consent (spec TRIGGERS-PLAN): apps BUILT here are @@ -2434,8 +2532,8 @@ async def _import_project_tree( # Runtime junk never imports; node_modules is skipped because a # foreign machine's install may not run here — the launch pipeline's # install step rebuilds it from package.json. .factory/.snapshots are - # the DONOR's lifecycle state (machine history, delivered flag, - # baseline) — a fresh identity must start a fresh lifecycle. + # the DONOR's lifecycle state (machine history, delivery stamp, + # legacy baseline) — a fresh identity must start a fresh lifecycle. shutil.copytree( src, dest, @@ -2503,8 +2601,9 @@ async def _import_project_tree( status="stopped", port=port, ) - # Delivered on arrival: an imported app may carry real data — later - # gates/verifies run in staging mode, never a baseline restore. + # Delivered on arrival: an imported app may carry real data. Its + # first boot creates/keeps its live pb_data, so later code changes + # run as modify arcs (dev env + promote) structurally. self._register_acquired(project, delivered=True) logger.info(f"[LIVING_UI] Imported project: {display} ({project_id})") @@ -2835,9 +2934,9 @@ async def install_from_marketplace( project.auto_launch = existing.auto_launch # Delivered on arrival (may ship with real data, never - # walk-verified): marked BEFORE the launch so the success path - # doesn't snapshot their pb_data as a "pristine" baseline and - # later verifies run in staging mode. + # walk-verified). The launch below creates its live pb_data, so + # later code changes run as modify arcs (dev env + promote) + # structurally. self._register_acquired(project, delivered=True) logger.info( @@ -3414,12 +3513,17 @@ async def stop_project(self, project_id: str) -> bool: logger.info(f"[LIVING_UI] Stopped project: {project_id}") return True - async def delete_project(self, project_id: str) -> bool: + async def delete_project( + self, project_id: str, delete_backups: bool = False + ) -> bool: """ Delete a Living UI project. Args: project_id: Project ID to delete + delete_backups: Also remove its pb_data backup archives. + Default KEEP (D5): backups exist precisely to outlive + mistakes, and deleting the app may be one. Returns: True if deletion was successful @@ -3429,6 +3533,14 @@ async def delete_project(self, project_id: str) -> bool: logger.error(f"[LIVING_UI] Project not found: {project_id}") return False + if delete_backups: + try: + self.backups.store.delete_project_backups(project_id) + except Exception as e: + logger.warning( + f"[LIVING_UI:BACKUP] backup cleanup failed for {project_id}: {e}" + ) + # Stop tunnel if active await self.stop_tunnel(project_id) @@ -3524,7 +3636,7 @@ def export_project_zip(self, project_id: str) -> Path: "logs", ".venv", "venv", - ".snapshots", # pristine pb_data baseline — local delivery state + ".snapshots", # legacy baseline dirs (pre-unified-lifecycle) — local state } skip_suffixes = {".pyc", ".pyo", ".log", ".db", ".sqlite", ".sqlite3"} skip_names = { diff --git a/app/living_ui/pb_data_io.py b/app/living_ui/pb_data_io.py index 8433c0d6..bc668a4e 100644 --- a/app/living_ui/pb_data_io.py +++ b/app/living_ui/pb_data_io.py @@ -1,15 +1,15 @@ """ -pb_data snapshot / restore — the data half of test-junk isolation. - -Two callers, two eras (spec: plans/quizzical-greeting-alpaca): - - build era: LivingUIManager snapshots a pristine baseline of pb_data at the - first successful launch and restores it right before the delivery - announce, so the user's first sight of the app has no agent/verifier - test records in it. Migration files added during the build are not in the - restored DB's _migrations table, so PocketBase re-applies them on boot — - schema survives the restore, junk doesn't. - - modify era: StagingSupervisor clones the live DB into a staging copy so - the gate/verifier never touch real user data. +pb_data snapshot / restore utilities. + +NO LIFECYCLE CODE CALLS THESE ANY MORE. The baseline-restore era (snapshot +at first launch, restore before the delivery announce) ended with the +unified dev/live lifecycle (docs/plans/living-ui-unified-lifecycle-plan.md) +after a stale delivered-flag made the restore wipe a live database +(2026-08-19). Dev environments boot with a FRESH pb_data instead — nothing +clones or restores over live data. + +The functions stay for the deferred pre-promote backup +(Promoter.add_before_live_boot_hook is the reserved slot) and for tooling. Copies go through sqlite's backup API, never shutil: PocketBase runs its DBs in WAL mode, and a naive file copy of a live data.db loses every write still diff --git a/app/living_ui/runner.py b/app/living_ui/runner.py index a5978c2b..74a050f9 100644 --- a/app/living_ui/runner.py +++ b/app/living_ui/runner.py @@ -26,6 +26,13 @@ INSTALL_TIMEOUT_S = 600 HEALTH_TIMEOUT_S = 30 +# The lui CLI is TypeScript executed by Node's native type stripping — +# default from 23.6, stable in 24. Older majors throw +# ERR_UNKNOWN_FILE_EXTENSION on cli.ts, which used to surface as a raw +# scaffold stack trace instead of this requirement (observed 2026-08-19, +# system Node 22.14). +MIN_NODE_MAJOR = 24 + @dataclass class V2ScaffoldResult: @@ -45,12 +52,34 @@ class LivingUIRunnerUnavailable(RuntimeError): """Node or the living-ui workspace is missing.""" +def read_superuser_creds(project_dir: Path): + """(email, password) from the project's 0600 `.superuser` file, or None + when the file is absent/unreadable/incomplete. The ONE parser of that + file — ensure_superuser writes it and reads through here; the backup + service reads through here to call the PocketBase admin API. Never log + the values.""" + import json as _json + + try: + stored = _json.loads( + (Path(project_dir) / ".superuser").read_text(encoding="utf-8") + ) + email = stored.get("email") or "" + password = stored.get("password") or "" + if email and password: + return (email, password) + except Exception: + pass + return None + + class LivingUIRunner: """Drives Living UI projects through scaffold → install → gate → serve.""" def __init__(self, workspace_dir: Path): self.workspace_dir = Path(workspace_dir) self._node = shutil.which("node") + self._node_version: Optional[str] = None # probed lazily, cached # ------------------------------------------------------------------ setup @@ -58,10 +87,47 @@ def __init__(self, workspace_dir: Path): def cli_path(self) -> Path: return self.workspace_dir / "tools" / "src" / "cli.ts" + def _probe_node_version(self) -> Optional[str]: + """`node --version` output ("v24.1.0"), cached. None when the probe + fails — version enforcement then fails open (a broken probe must + never block a launch on a good Node).""" + if self._node_version is not None: + return self._node_version + try: + kwargs = {} + if sys.platform == "win32": + kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW + out = subprocess.run( + [self._node, "--version"], + capture_output=True, + text=True, + timeout=15, + **kwargs, + ).stdout.strip() + if out: + self._node_version = out + except Exception as e: + logger.warning(f"node version probe failed: {e}") + return self._node_version + def ensure_available(self) -> None: if self._node is None: raise LivingUIRunnerUnavailable( - "Node.js >= 24 is required to build Living UIs (not found on PATH)." + f"Node.js >= {MIN_NODE_MAJOR} is required to build Living UIs " + "(not found on PATH)." + ) + version = self._probe_node_version() + try: + major = int((version or "").lstrip("v").split(".")[0]) + except ValueError: + major = None + if major is not None and major < MIN_NODE_MAJOR: + raise LivingUIRunnerUnavailable( + f"Node.js >= {MIN_NODE_MAJOR} is required to build Living UIs — " + f"found {version} at {self._node}. The lui CLI is TypeScript " + "run natively by Node (type stripping), which this version " + "cannot load. Upgrade Node (or install nodejs>=24 into the " + "conda env CraftBot runs in) and restart." ) if not self.cli_path.exists(): raise LivingUIRunnerUnavailable( @@ -251,17 +317,9 @@ async def ensure_superuser(self, project_dir: Path) -> None: pb_dir = project_dir / "pb" cred_file = project_dir / ".superuser" - email = "agent@lui.local" - password = "" - if cred_file.exists(): - try: - stored = _json.loads(cred_file.read_text(encoding="utf-8")) - email = stored.get("email") or email - password = stored.get("password") or "" - except Exception: - password = "" - if password == "": - password = secrets.token_urlsafe(18) + creds = read_superuser_creds(project_dir) + email = creds[0] if creds else "agent@lui.local" + password = creds[1] if creds else secrets.token_urlsafe(18) code, out = await self._run( [ diff --git a/app/living_ui/staging.py b/app/living_ui/staging.py deleted file mode 100644 index 348df8da..00000000 --- a/app/living_ui/staging.py +++ /dev/null @@ -1,329 +0,0 @@ -""" -Staging copies of DELIVERED Living UI apps — the modify-era half of -test-junk isolation (spec: plans/quizzical-greeting-alpaca). - -Once an app is delivered its pb_data holds real user data AND PocketBase -serves the frontend from disk per-request (--publicDir pb/pb_public, which -the gate's vite build overwrites with emptyOutDir: true). So on a delivered -app, running the gate against the real directory blanks the live UI, and -letting the agent/verifier test against the real port pollutes real data. - -The staging copy fixes both mechanically: a full project copy under -living_ui/_staging/project// with a cloned DB, booted on a hidden port. -All gating, relaunching, agent testing and walk-verification happen there; -the real app keeps serving the old working code untouched. On a clean -verify, the caller "flips" — relaunches the real project (new migrations -apply to real data at boot) and destroys the copy, and every test record -dies with it. - -Composition mirrors LivingUIRunner: the manager constructs and drives this class; -it never reaches back into the manager or the registry. The authoritative -"a staging copy exists" record lives in the factory host sidecar -(.factory/host.json, key "staging") — actions redirect from it, the boot -reaper kills from it, clearing it ends staging mode. -""" - -import json -import os -import re -import shutil -import signal -import socket -import subprocess -import time -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Dict, Optional - -try: - from loguru import logger -except ImportError: - import logging - - logger = logging.getLogger(__name__) - -from app.living_ui.pb_data_io import snapshot_pb_data - -# Outside the manager's 3100-3199 pool on purpose: _load_projects rebuilds -# port bookkeeping from registered projects only, and cleanup_on_startup's -# orphan killer scans that range — staging owns its ports and its reaping. -STAGING_PORT_RANGE = (3900, 3999) - -# Same guard the wizard uses for its staging ids: nothing outside this -# pattern ever becomes part of an rmtree'd path. -_ID_RE = re.compile(r"^[A-Za-z0-9_-]{4,64}$") - -# What a staging copy takes from the real project. pb_data arrives via the -# sqlite backup API (never a file copy of a live WAL DB); pb_public is -# deliberately absent — the gate's build step recreates it inside the copy. -# triggers.json MUST travel: without it the copy's trigger guard declares -# nothing, every ⚡ fire 400s, the walker fails an unfixable "defect", and -# the arc sticks (observed live 2026-08-06, kanban board — three identical -# STUCKs on one missing file). -_COPY_FILES = ("manifest.json", "operations.json", "triggers.json", "LIVING_UI.md") -_COPY_CREDS = (".superuser", ".agent-token") -_COPY_DIRS = ("frontend", "pb/pb_hooks", "pb/pb_migrations", ".lui", "reference") - -# What sync_code refreshes on each fix-mission iteration: the agent-owned -# paths (ownership rule, agent-guide §1) — never manifest.json (the copy's -# port rewrite must survive) and never pb/pb_data (the agent's in-app test -# data persists across iterations). -_SYNC_FILES = ("operations.json", "triggers.json", "LIVING_UI.md") -_SYNC_DIRS = ("frontend/src", "pb/pb_hooks", "pb/pb_migrations", "reference") -_SYNC_PKG = ("frontend/package.json", "frontend/package-lock.json") - - -@dataclass -class StagingInstance: - """One staging copy. `process` is runtime-only; everything else - round-trips through the sidecar record.""" - - project_id: str - dir: Path - port: int - created_at: float - pid: Optional[int] = None - process: Optional[subprocess.Popen] = None - - @property - def url(self) -> str: - return f"http://127.0.0.1:{self.port}" - - def to_record(self) -> Dict[str, Any]: - return { - "dir": str(self.dir), - "port": self.port, - "url": self.url, - "pid": self.pid, - "created_at": self.created_at, - } - - @classmethod - def from_record(cls, project_id: str, record: Dict[str, Any]) -> "StagingInstance": - return cls( - project_id=project_id, - dir=Path(record.get("dir", "")), - port=int(record.get("port", 0)), - created_at=float(record.get("created_at", 0)), - pid=record.get("pid"), - ) - - -class StagingSupervisor: - """Creates, refreshes, destroys and reaps staging copies. Knows nothing - about the manager's registry, sessions or broadcasting — the manager - composes this class; it never reaches back.""" - - def __init__(self, living_ui_dir: Path, runner) -> None: - self.living_ui_dir = Path(living_ui_dir) - self.root = self.living_ui_dir / "_staging" / "project" - self.runner = runner - # Live process handles, keyed by project id. Best-effort only — - # after a CraftBot restart the pid in the sidecar record is all - # that's left, and destroy/reap fall back to it. - self._processes: Dict[str, subprocess.Popen] = {} - - # ── create / refresh ─────────────────────────────────────────────────── - async def create_copy(self, project) -> StagingInstance: - """Build a fresh staging copy of `project` (code + DB clone) and - rewrite its identity for a hidden port. Does NOT boot it — the - manager runs the shared launch pipeline against the returned dir. - Raises on failure; a partial copy is removed.""" - if not _ID_RE.match(project.id or ""): - raise ValueError(f"unsafe project id for staging: {project.id!r}") - src = Path(project.path) - if not (src / "manifest.json").exists(): - raise FileNotFoundError(f"not a Living UI project: {src}") - - staging_dir = self.root / project.id - if staging_dir.exists(): - self._guarded_rmtree(staging_dir) - staging_dir.mkdir(parents=True) - - try: - for rel in _COPY_FILES + _COPY_CREDS: - f = src / rel - if f.exists(): - shutil.copy2(f, staging_dir / rel) - for rel in _COPY_DIRS: - d = src / rel - if d.is_dir(): - # node_modules rides along inside frontend/ — without it - # the gate cold-installs for up to 600 s per staging boot. - shutil.copytree(d, staging_dir / rel, symlinks=True) - (staging_dir / "logs").mkdir(exist_ok=True) - - # DB clone: consistent even while the real app is serving. - snapshot_pb_data( - src / "pb" / "pb_data", - staging_dir / "pb" / "pb_data", - self.living_ui_dir, - ) - - port = self._free_port() - self._rewrite_manifest_port(staging_dir, port) - - # The port rewrite invalidated the system-hash canon; kit-sync - # re-vendors the kit and re-records hashes (same recovery the - # ZIP-import path uses) — without it the gate's ownership step - # fails with "modified: manifest.json". - await self.runner.kit_sync(staging_dir) - except Exception: - self._guarded_rmtree(staging_dir) - raise - - instance = StagingInstance( - project_id=project.id, - dir=staging_dir, - port=port, - created_at=time.time(), - ) - logger.info( - f"[LIVING_UI:STAGING] created copy of {project.id} at " - f"{staging_dir} (port {port})" - ) - return instance - - def sync_code(self, project, staging_dir: Path) -> None: - """Refresh the agent-owned paths real → staging (fix-mission - iterations edit the real files; the staging copy is what gets gated - and served). Keeps staging pb_data and the rewritten manifest.""" - src = Path(project.path) - staging_dir = Path(staging_dir) - if not (staging_dir / "manifest.json").exists(): - raise FileNotFoundError(f"staging copy missing at {staging_dir}") - - # A changed package.json means new/changed deps: drop node_modules so - # the pipeline's install step runs for real instead of being skipped. - for rel in _SYNC_PKG: - s, d = src / rel, staging_dir / rel - if s.exists() and (not d.exists() or s.read_bytes() != d.read_bytes()): - shutil.copy2(s, d) - nm = staging_dir / "frontend" / "node_modules" - if nm.is_dir(): - logger.info( - "[LIVING_UI:STAGING] package.json changed — " - "clearing staging node_modules for a fresh install" - ) - self._guarded_rmtree(nm) - - for rel in _SYNC_FILES: - s = src / rel - if s.exists(): - shutil.copy2(s, staging_dir / rel) - for rel in _SYNC_DIRS: - s, d = src / rel, staging_dir / rel - if s.is_dir(): - if d.exists(): - self._guarded_rmtree(d) - shutil.copytree(s, d, symlinks=True) - - # ── process bookkeeping ──────────────────────────────────────────────── - def adopt_process(self, instance: StagingInstance, process) -> None: - instance.process = process - instance.pid = process.pid - self._processes[instance.project_id] = process - - # ── destroy / reap ───────────────────────────────────────────────────── - def destroy(self, project_id: str, record: Optional[Dict[str, Any]]) -> None: - """Kill the staging process and delete the copy. Idempotent and - best-effort: a half-dead staging must never block a flip.""" - process = self._processes.pop(project_id, None) - if process is not None and process.poll() is None: - self._kill(process=process) - elif record and record.get("pid"): - self._kill(pid=int(record["pid"])) - - staging_dir = ( - Path(record["dir"]) - if record and record.get("dir") - else (self.root / project_id) - ) - if staging_dir.exists(): - try: - self._guarded_rmtree(staging_dir) - logger.info(f"[LIVING_UI:STAGING] destroyed copy of {project_id}") - except Exception as e: - logger.warning( - f"[LIVING_UI:STAGING] failed to delete {staging_dir}: {e}" - ) - - def reap_all(self, records: Dict[str, Dict[str, Any]]) -> int: - """Startup reaper: no staging copy is legitimately alive when - CraftBot boots (their missions died with the process), so kill every - recorded pid and delete everything under the staging root — including - dirs with no surviving record. Deliberate, unlike the blind orphan - rmtree in cleanup_on_startup (which skips _staging entirely).""" - reaped = 0 - for project_id, record in records.items(): - self.destroy(project_id, record) - reaped += 1 - if self.root.exists(): - for leftover in self.root.iterdir(): - try: - self._guarded_rmtree(leftover) - reaped += 1 - logger.info(f"[LIVING_UI:STAGING] reaped leftover {leftover.name}") - except Exception as e: - logger.warning( - f"[LIVING_UI:STAGING] failed to reap {leftover}: {e}" - ) - return reaped - - # ── internals ────────────────────────────────────────────────────────── - def _guarded_rmtree(self, target: Path) -> None: - """Only ever delete inside living_ui/_staging/ — the same - strict-ancestor discipline delete_project adopted after rmtree wiped - the working tree twice (2026-07-25/26).""" - resolved = Path(target).resolve() - staging_root = (self.living_ui_dir / "_staging").resolve() - if staging_root not in resolved.parents: - raise ValueError(f"refusing to delete {resolved} — outside {staging_root}") - shutil.rmtree(resolved) - - def _free_port(self) -> int: - for port in range(STAGING_PORT_RANGE[0], STAGING_PORT_RANGE[1] + 1): - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - s.bind(("127.0.0.1", port)) - return port - except OSError: - continue - raise RuntimeError("No free port in the staging range 3900-3999") - - def _rewrite_manifest_port(self, staging_dir: Path, port: int) -> None: - """`lui ops/run/data` derive their base URL from manifest.port — a - stale port would make CLI calls from the staging dir hit the LIVE - app. Same rewrite (including the inlined pipeline string) the - ZIP-import path does.""" - manifest_path = staging_dir / "manifest.json" - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - old_port = manifest.get("port") - manifest["port"] = port - if isinstance(manifest.get("pipeline"), dict) and old_port: - manifest["pipeline"] = json.loads( - json.dumps(manifest["pipeline"]).replace(str(old_port), str(port)) - ) - manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") - - def _kill(self, process=None, pid: Optional[int] = None) -> None: - try: - if process is not None: - process.terminate() - try: - process.wait(timeout=5) - except Exception: - process.kill() - elif pid: - os.kill(pid, signal.SIGTERM) - time.sleep(0.5) - try: - os.kill(pid, 0) - except OSError: - return # already gone - os.kill(pid, signal.SIGKILL) - except ProcessLookupError: - pass - except Exception as e: - logger.warning(f"[LIVING_UI:STAGING] kill failed: {e}") diff --git a/app/living_ui/test_backups.py b/app/living_ui/test_backups.py new file mode 100644 index 00000000..06ff7c66 --- /dev/null +++ b/app/living_ui/test_backups.py @@ -0,0 +1,645 @@ +"""Acceptance tests for Living UI backups — store, capture (both paths), +scheduler, pre-promote hook, restore, boot-cleaner/delete integration, +settings surface, and a real-PocketBase end-to-end (§8, skipped when the +pinned binary is not cached). + +Spec: docs/plans/living-ui-backups-requirements.md / -plan.md. + +Style follows test_data_safety.py: a module-level assert script with +section prints — run directly: + + PYTHONPATH=. python app/living_ui/test_backups.py +""" + +import asyncio +import sqlite3 +import tempfile +import time +import zipfile +from pathlib import Path +from types import SimpleNamespace + +import app.factory.host_craftbot as host_mod +import app.living_ui as living_ui_mod +from app.living_ui.lifecycle.backups import ( + _NAME_RE, + PRE_PROMOTE_KEEP, + BackupService, + BackupStore, + _ts_name, +) +from app.living_ui.manager import LivingUIManager, LivingUIProject +from app.living_ui.pb_data_io import restore_pb_data + + +# ── shared helpers ───────────────────────────────────────────────────────── +def _mkdb(path: Path, rows: int, table: str = "items") -> None: + path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(path) + with con: + con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id INTEGER PRIMARY KEY)") + con.execute(f"DELETE FROM {table}") + con.executemany( + f"INSERT INTO {table} (id) VALUES (?)", [(i,) for i in range(1, rows + 1)] + ) + con.close() + + +def _count(path: Path, table: str = "items") -> int: + con = sqlite3.connect(path) + try: + return con.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + finally: + con.close() + + +def _touch_archive(store: BackupStore, pid: str, ts: float, trigger: str) -> Path: + p = store.project_dir(pid) / f"{_ts_name(ts)}__{trigger}.zip" + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(b"zip" + bytes(int(ts) % 251)) + return p + + +def _make_project(living: Path, pid: str) -> SimpleNamespace: + proj = living / f"app_{pid}" + pb_data = proj / "pb" / "pb_data" + _mkdb(pb_data / "data.db", 3) + _mkdb(pb_data / "auxiliary.db", 1, table="aux") + (pb_data / "types.d.ts").write_text("// generated\n") + (pb_data / "data.db-wal").write_bytes(b"") + (pb_data / "storage" / "rec1").mkdir(parents=True) + (pb_data / "storage" / "rec1" / "upload.bin").write_bytes(b"file-bytes") + (pb_data / "backups" / "old.zip").parent.mkdir(parents=True, exist_ok=True) + (pb_data / "backups" / "old.zip").write_bytes(b"stale pb-native backup") + return SimpleNamespace(id=pid, path=str(proj), status="stopped", port=0) + + +# ── §1 BackupStore: naming, pools, prune, guards ─────────────────────────── +with tempfile.TemporaryDirectory() as tmp: + living = Path(tmp) / "living_ui" + store = BackupStore(living) + + # canonical naming round-trips; claim_path bumps same-second collisions + p1 = store.claim_path("proj0001", "scheduled", ts=1000000.0) + p1.write_bytes(b"a") + p2 = store.claim_path("proj0001", "scheduled", ts=1000000.0) + assert p1 != p2 and _NAME_RE.match(p2.name), "collision must bump, stay canonical" + p2.write_bytes(b"b") + + # unsafe ids refused before any path is built + for bad in ("", "..", "a/b", "x" * 65): + try: + store.project_dir(bad) + raise AssertionError(f"id {bad!r} must be refused") + except ValueError: + pass + + # listing: newest first, pools attributed, foreign files invisible + _touch_archive(store, "proj0001", 2000.0, "pre_promote") + _touch_archive(store, "proj0001", 3000.0, "manual") + (store.project_dir("proj0001") / "README.txt").write_text("not a backup") + (store.project_dir("proj0001") / "20990101T000000Z__evil.zip").write_bytes(b"x") + entries = store.list_backups("proj0001") + assert [e.trigger for e in entries] == [ + "scheduled", + "scheduled", + "manual", + "pre_promote", + ] + assert entries[0].ts >= entries[1].ts + assert store.total_size("proj0001") == sum(e.size for e in entries) + + # prune: only the named pool shrinks; foreign files untouched + for ts in (10.0, 20.0, 30.0, 40.0): + _touch_archive(store, "proj0002", ts, "scheduled") + _touch_archive(store, "proj0002", 15.0, "manual") + assert store.prune("proj0002", "scheduled", keep=2) == 2 + kept = store.list_backups("proj0002") + assert [e.ts for e in kept if e.trigger == "scheduled"] == [40.0, 30.0] + assert [e.ts for e in kept if e.trigger == "manual"] == [15.0], ( + "other pools survive" + ) + assert store.prune("proj0002", "scheduled", keep=2) == 0, "idempotent" + + # delete: canonical names only; guard refuses paths outside the root + store.delete("proj0001", entries[-1].filename) + for bad_name in ("../evil.zip", "README.txt", "x.zip", ""): + try: + store.delete("proj0001", bad_name) + raise AssertionError(f"delete must refuse {bad_name!r}") + except ValueError: + pass + outside = Path(tmp) / "outside.txt" + outside.write_text("precious") + try: + store._guarded_delete(outside) + raise AssertionError("guard must refuse targets outside _backups") + except ValueError: + pass + assert outside.exists() + assert (store.project_dir("proj0001") / "README.txt").exists() + assert (store.project_dir("proj0001") / "20990101T000000Z__evil.zip").exists() + + # delete_project_backups removes the dir; orphans are reported, not reaped + store.delete_project_backups("proj0002") + assert not store.project_dir("proj0002").exists() + assert store.orphan_dirs(registered_ids=[]) == ["proj0001"] + assert store.orphan_dirs(registered_ids=["proj0001"]) == [] +print("§1 BackupStore naming/pools/prune/guards: OK") + + +# ── §2 capture_stopped round-trip ────────────────────────────────────────── +with tempfile.TemporaryDirectory() as tmp: + living = Path(tmp) / "living_ui" + living.mkdir() + svc = BackupService(living) + project = _make_project(living, "cap00001") + pb_data = Path(project.path) / "pb" / "pb_data" + + entry = svc.capture_stopped(project, "manual") + assert entry.path.exists() and entry.trigger == "manual" and entry.size > 0 + assert not (entry.path.parent / ".tmp").exists(), "capture temp must be cleaned" + assert not list(entry.path.parent.glob("*.part")), "no partial archives" + + # archive carries DBs + storage, excludes PB-native backups/, WAL, types + names = set(zipfile.ZipFile(entry.path).namelist()) + assert "data.db" in names and "auxiliary.db" in names + assert "storage/rec1/upload.bin" in names + assert not any(n.startswith("backups") for n in names) + assert not any(n.endswith((".d.ts", "-wal")) for n in names) + + # post-backup junk rows vanish on restore; storage bytes survive + _mkdb(pb_data / "data.db", 9) + (pb_data / "storage" / "rec1" / "junk.bin").write_bytes(b"junk") + unpack = living / "_backups" / "cap00001" / ".restore-tmp" + with zipfile.ZipFile(entry.path) as zf: + zf.extractall(unpack) + restore_pb_data(unpack, pb_data, living) + assert _count(pb_data / "data.db") == 3, "restore must drop post-backup junk" + assert (pb_data / "storage" / "rec1" / "upload.bin").read_bytes() == b"file-bytes" + assert not (pb_data / "storage" / "rec1" / "junk.bin").exists() + + # a second capture the same second lands beside the first, not over it + entry2 = svc.capture_stopped(project, "manual") + assert entry2.path != entry.path and len(svc.store.list_backups("cap00001")) == 2 + + # capture with no data.db raises and leaves nothing behind + bare = _make_project(living, "bare0001") + (Path(bare.path) / "pb" / "pb_data" / "data.db").unlink() + before = len(svc.store.list_backups("bare0001")) + try: + svc.capture_stopped(bare, "scheduled") + raise AssertionError("capture without data.db must raise") + except FileNotFoundError: + pass + assert len(svc.store.list_backups("bare0001")) == before + assert not (svc.store.project_dir("bare0001") / ".tmp").exists() +print("§2 capture_stopped round-trip: OK") + +# ── §3 scheduler due-logic ───────────────────────────────────────────────── +with tempfile.TemporaryDirectory() as tmp: + mgr = LivingUIManager(workspace_root=Path(tmp)) + project = _make_project(mgr.living_ui_dir, "sched001") + lp = LivingUIProject( + id="sched001", name="s", description="", path=project.path, status="stopped" + ) + mgr.projects["sched001"] = lp + living_ui_mod.get_living_ui_manager = lambda: mgr + host_mod._HOST = None + host = host_mod.get_factory_host() + + fired = [] + + async def _fake_run(p): + fired.append(p.id) + mgr._backups_inflight.discard(p.id) + + mgr._run_scheduled_backup = _fake_run + + async def _t3(): + # absent last_at -> due now (first-enable and boot catch-up) + mgr._maybe_schedule_backup(lp) + await asyncio.sleep(0) + assert fired == ["sched001"], "no last_at must mean due" + + # recent last_at -> not due; ancient -> due again + host.record_backup_ok("sched001", time.time()) + mgr._maybe_schedule_backup(lp) + assert len(fired) == 1, "fresh backup must not be due" + host.record_backup_ok("sched001", time.time() - 86400 - 5) + mgr._maybe_schedule_backup(lp) + await asyncio.sleep(0) + assert len(fired) == 2, "older than the interval must be due" + + # interval enum honored (hourly with a 2h-old stamp is due) + lp.backup_interval = "hourly" + host.record_backup_ok("sched001", time.time() - 7200) + mgr._maybe_schedule_backup(lp) + await asyncio.sleep(0) + assert len(fired) == 3 + + # gates: disabled / external / mid-op / inflight / no live DB + host.record_backup_ok("sched001", time.time() - 7200) + lp.backups_enabled = False + mgr._maybe_schedule_backup(lp) + lp.backups_enabled = True + lp.project_type = "external" + mgr._maybe_schedule_backup(lp) + lp.project_type = "native" + mgr._live_ops.add("sched001") + mgr._maybe_schedule_backup(lp) + mgr._live_ops.discard("sched001") + mgr._backups_inflight.add("sched001") + mgr._maybe_schedule_backup(lp) + mgr._backups_inflight.discard("sched001") + db = Path(lp.path) / "pb" / "pb_data" / "data.db" + db.rename(db.with_name("data.db.away")) + mgr._maybe_schedule_backup(lp) + db.with_name("data.db.away").rename(db) + await asyncio.sleep(0) + assert len(fired) == 3, "every gate must hold" + + # the real runner: capture + prune-to-keep + sidecar ok (clears error) + del mgr._run_scheduled_backup # back to the bound method + host.record_backup_error("sched001", "previous failure") + lp.backup_keep = 1 + for ts in (100.0, 200.0): + _touch_archive(mgr.backups.store, "sched001", ts, "scheduled") + mgr._backups_inflight.add("sched001") + await mgr._run_scheduled_backup(lp) + pool = [ + e + for e in mgr.backups.store.list_backups("sched001") + if e.trigger == "scheduled" + ] + assert len(pool) == 1 and pool[0].ts > 200.0, "prune to keep=1, newest wins" + state = host.backup_state("sched001") + assert state["last_at"] is not None and state["last_error"] is None, ( + "success must stamp last_at and clear last_error" + ) + assert "sched001" not in mgr._backups_inflight + + # a failing capture records the error and never raises out + db.rename(db.with_name("data.db.away")) + mgr._backups_inflight.add("sched001") + await mgr._run_scheduled_backup(lp) + db.with_name("data.db.away").rename(db) + assert host.backup_state("sched001")["last_error"], "failure must be recorded" + assert "sched001" not in mgr._backups_inflight + + asyncio.run(_t3()) +print("§3 scheduler due-logic: OK") + + +# ── §4 pre-promote hook ──────────────────────────────────────────────────── +with tempfile.TemporaryDirectory() as tmp: + mgr = LivingUIManager(workspace_root=Path(tmp)) + project = _make_project(mgr.living_ui_dir, "promo001") + lp = LivingUIProject( + id="promo001", name="p", description="", path=project.path, status="stopped" + ) + mgr.projects["promo001"] = lp + living_ui_mod.get_living_ui_manager = lambda: mgr + host_mod._HOST = None + host = host_mod.get_factory_host() + + async def _fake_launch_live(pid): + return {"status": "success", "url": "http://127.0.0.1:1", "port": 1} + + mgr.lifecycle.promoter._launch_live = _fake_launch_live + + # promote over a live DB captures pre_promote, prunes to the constant, + # and resets the scheduled clock + for ts in (10.0, 20.0, 30.0): + _touch_archive(mgr.backups.store, "promo001", ts, "pre_promote") + res = asyncio.run(mgr.promote("promo001")) + assert res["status"] == "success" and res["first"] is False + pool = [ + e + for e in mgr.backups.store.list_backups("promo001") + if e.trigger == "pre_promote" + ] + assert len(pool) == PRE_PROMOTE_KEEP and pool[0].ts > 30.0, ( + "hook must capture and prune to the constant" + ) + assert host.backup_state("promo001")["last_at"] is not None + assert "promo001" not in mgr._live_ops + + # first delivery (no live DB): the hook no-ops, promote proceeds + db = Path(lp.path) / "pb" / "pb_data" / "data.db" + db.unlink() + n_before = len(mgr.backups.store.list_backups("promo001")) + res = asyncio.run(mgr.promote("promo001")) + assert res["status"] == "success" and res["first"] is True + assert len(mgr.backups.store.list_backups("promo001")) == n_before, ( + "no live DB -> nothing to protect -> no archive" + ) + _mkdb(db, 3) + + # a raising capture ABORTS the promote before the live boot + def _boom(project, trigger): + raise RuntimeError("disk full") + + mgr.backups.capture_stopped = _boom + res = asyncio.run(mgr.promote("promo001")) + assert res["status"] == "error" and res["step"] == "before_live_boot", ( + "failed pre-promote backup must abort the promote" + ) + assert "promo001" not in mgr._live_ops, "mid-op marker must clear on abort" +print("§4 pre-promote hook: OK") + +# ── §5 restore round-trip ────────────────────────────────────────────────── +with tempfile.TemporaryDirectory() as tmp: + mgr = LivingUIManager(workspace_root=Path(tmp)) + project = _make_project(mgr.living_ui_dir, "rest0001") + lp = LivingUIProject( + id="rest0001", + name="r", + description="", + path=project.path, + status="running", + port=3131, + ) + mgr.projects["rest0001"] = lp + living_ui_mod.get_living_ui_manager = lambda: mgr + host_mod._HOST = None + host = host_mod.get_factory_host() + + RELAUNCHES = [] + + async def _fake_relaunch(pid): + RELAUNCHES.append(pid) + lp.status = "running" + return {"status": "success", "url": "http://127.0.0.1:3131"} + + mgr.launch_and_verify = _fake_relaunch + db = Path(lp.path) / "pb" / "pb_data" / "data.db" + + async def _t5(): + # take a backup at 3 rows, then "life happens": 8 rows + a new file + entry = await asyncio.to_thread(mgr.backups.capture_stopped, lp, "manual") + _mkdb(db, 8) + (db.parent / "storage" / "rec1" / "later.bin").write_bytes(b"post-backup") + + res = await mgr.restore_backup("rest0001", entry.filename) + assert res["status"] == "success" and res["restored"] == entry.filename + assert _count(db) == 3, "restore must return to the archived state" + assert not (db.parent / "storage" / "rec1" / "later.bin").exists() + assert RELAUNCHES == ["rest0001"], "restored app must relaunch" + assert "rest0001" not in mgr._live_ops + + # ...and it is REVERSIBLE: the pre-restore capture holds the 8 rows + pre_name = res["pre_restore_backup"] + manual_pool = [ + e + for e in mgr.backups.store.list_backups("rest0001") + if e.trigger == "manual" + ] + assert any(e.filename == pre_name for e in manual_pool) + res2 = await mgr.restore_backup("rest0001", pre_name) + assert res2["status"] == "success" + assert _count(db) == 8, "restoring the pre-restore backup must undo" + assert not (mgr.backups.store.project_dir("rest0001") / ".restore-tmp").exists() + + # unknown archive / external app / mid-op are refused up front + res = await mgr.restore_backup("rest0001", "20990101T000000Z__manual.zip") + assert res["status"] == "error" + lp.project_type = "external" + assert (await mgr.restore_backup("rest0001", pre_name))["status"] == "error" + lp.project_type = "native" + mgr._live_ops.add("rest0001") + res = await mgr.restore_backup("rest0001", pre_name) + assert res["status"] == "error" and "in flight" in res["errors"][0] + mgr._live_ops.discard("rest0001") + + # FR9 2a: failing pre-restore capture ABORTS with data untouched + real_capture = mgr.backups.capture_stopped + + def _boom(project, trigger): + raise RuntimeError("no space") + + mgr.backups.capture_stopped = _boom + rows_before = _count(db) + res = await mgr.restore_backup("rest0001", pre_name) + assert res["status"] == "error" and res["step"] == "pre_restore_backup" + assert _count(db) == rows_before, "aborted restore must not touch pb_data" + mgr.backups.capture_stopped = real_capture + assert "rest0001" not in mgr._live_ops + + asyncio.run(_t5()) +print("§5 restore round-trip: OK") + +# ── §6 boot cleaner + delete-project integration ─────────────────────────── +with tempfile.TemporaryDirectory() as tmp: + mgr = LivingUIManager(workspace_root=Path(tmp)) + living_ui_mod.get_living_ui_manager = lambda: mgr + host_mod._HOST = None + + project = _make_project(mgr.living_ui_dir, "keep0001") + lp = LivingUIProject( + id="keep0001", name="k", description="", path=project.path, status="stopped" + ) + mgr.projects["keep0001"] = lp + _touch_archive(mgr.backups.store, "keep0001", 100.0, "scheduled") + _touch_archive(mgr.backups.store, "gone0001", 100.0, "manual") # orphan's + (mgr.living_ui_dir / "app_orphan_project").mkdir() + + # the boot cleaner LOGS unregistered dirs but NEVER deletes them, and + # always skips _backups/_staging + (mgr.living_ui_dir / "_staging").mkdir(exist_ok=True) + found = mgr._log_orphan_folders() + assert found == 1 + assert (mgr.living_ui_dir / "app_orphan_project").exists(), ( + "boot cleaner must NEVER delete orphan folders (only log them)" + ) + assert (mgr.living_ui_dir / "_backups").exists(), ( + "boot cleaner must NEVER touch _backups" + ) + assert mgr.backups.store.list_backups("keep0001"), "archives must survive boot" + + # delete_project default: project dir dies, backups become a listed orphan + asyncio.run(mgr.delete_project("keep0001")) + assert "keep0001" not in mgr.projects and not Path(project.path).exists() + assert mgr.backups.store.list_backups("keep0001"), ( + "default delete must KEEP backups (D5)" + ) + assert set(mgr.backups.store.orphan_dirs(mgr.projects.keys())) == { + "keep0001", + "gone0001", + } + + # delete_project with delete_backups=True removes the archives too + project2 = _make_project(mgr.living_ui_dir, "kill0001") + lp2 = LivingUIProject( + id="kill0001", name="k2", description="", path=project2.path, status="stopped" + ) + mgr.projects["kill0001"] = lp2 + _touch_archive(mgr.backups.store, "kill0001", 100.0, "scheduled") + asyncio.run(mgr.delete_project("kill0001", delete_backups=True)) + assert not mgr.backups.store.project_dir("kill0001").exists() +print("§6 boot cleaner + delete-project: OK") + + +# ── §7 settings surface ──────────────────────────────────────────────────── +from app.ui_layer.settings.living_ui_settings import ( # noqa: E402 + get_living_ui_projects, + update_project_setting, +) + +with tempfile.TemporaryDirectory() as tmp: + mgr = LivingUIManager(workspace_root=Path(tmp)) + project = _make_project(mgr.living_ui_dir, "sett0001") + lp = LivingUIProject( + id="sett0001", name="s", description="", path=project.path, status="stopped" + ) + mgr.projects["sett0001"] = lp + living_ui_mod.get_living_ui_manager = lambda: mgr + host_mod._HOST = None + + # DTO carries the backup settings + status + orphans + _touch_archive(mgr.backups.store, "sett0001", 100.0, "manual") + _touch_archive(mgr.backups.store, "olddead1", 100.0, "manual") + out = get_living_ui_projects() + assert out["success"] and out["backupOrphans"] == ["olddead1"] + dto = out["projects"][0] + assert dto["backupsEnabled"] is True and dto["backupInterval"] == "daily" + assert dto["backupKeep"] == 7 and dto["backupStatus"]["count"] == 1 + assert dto["projectType"] == "native" + + # validation branches + assert update_project_setting("sett0001", "backupsEnabled", False)["success"] + assert lp.backups_enabled is False + assert update_project_setting("sett0001", "backupInterval", "weekly")["success"] + assert not update_project_setting("sett0001", "backupInterval", "monthly")[ + "success" + ] + assert not update_project_setting("sett0001", "backupKeep", "abc")["success"] + assert not update_project_setting("sett0001", "backupKeep", 0)["success"] + assert not update_project_setting("sett0001", "backupKeep", 31)["success"] + assert not update_project_setting("sett0001", "nope", 1)["success"] + + # prune-on-shrink: lowering keep applies immediately + for ts in (10.0, 20.0, 30.0, 40.0): + _touch_archive(mgr.backups.store, "sett0001", ts, "scheduled") + assert update_project_setting("sett0001", "backupKeep", 2)["success"] + pool = [ + e + for e in mgr.backups.store.list_backups("sett0001") + if e.trigger == "scheduled" + ] + assert [e.ts for e in pool] == [40.0, 30.0], "shrink must prune immediately" +print("§7 settings surface: OK") + +# ── §8 capture_running against a REAL PocketBase ─────────────────────────── +# Uses the pinned binary from the lui cache (fetched by any prior Living UI +# build on this machine). Skipped when absent — §1-§7 stay deterministic. +import json as _json # noqa: E402 +import os as _os # noqa: E402 +import socket as _socket # noqa: E402 +import subprocess as _sp # noqa: E402 +import time as _time # noqa: E402 +import urllib.request as _url # noqa: E402 + + +def _pinned_pb_binary() -> Path: + version = ( + ( + Path(__file__).resolve().parents[2] + / "living-ui" + / "spec" + / "pocketbase.version" + ) + .read_text(encoding="utf-8") + .strip() + ) + cache = _os.environ.get("LIVING_UI_PB_CACHE") + if cache: + root = Path(cache) + elif _os.name == "nt": + root = Path(_os.environ["LOCALAPPDATA"]) / "craftos-living-ui" / "pb" + else: + root = Path.home() / ".cache" / "craftos-living-ui" / "pb" + exe = "pocketbase.exe" if _os.name == "nt" else "pocketbase" + return root / version / exe + + +_pb_bin = _pinned_pb_binary() +if not _pb_bin.exists(): + print(f"§8 capture_running vs real PocketBase: SKIPPED (no binary at {_pb_bin})") +else: + with tempfile.TemporaryDirectory() as tmp: + living = Path(tmp) / "living_ui" + proj_dir = living / "app_pbe2e001" + pb_data = proj_dir / "pb" / "pb_data" + pb_data.mkdir(parents=True) + email, password = "agent@lui.local", "e2e-test-password-123" + assert ( + _sp.run( + [ + str(_pb_bin), + "superuser", + "upsert", + email, + password, + "--dir", + str(pb_data), + ], + capture_output=True, + timeout=60, + ).returncode + == 0 + ), "superuser upsert failed" + (proj_dir / ".superuser").write_text( + _json.dumps({"email": email, "password": password}) + "\n" + ) + + with _socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + proc = _sp.Popen( + [str(_pb_bin), "serve", f"--http=127.0.0.1:{port}", "--dir", str(pb_data)], + stdout=_sp.DEVNULL, + stderr=_sp.DEVNULL, + ) + try: + for _ in range(100): # wait_healthy, poor man's edition + try: + _url.urlopen(f"http://127.0.0.1:{port}/api/health", timeout=1) + break + except Exception: + _time.sleep(0.2) + else: + raise AssertionError("PocketBase never became healthy") + + svc = BackupService(living) + project = SimpleNamespace( + id="pbe2e001", path=str(proj_dir), status="running", port=port + ) + entry = asyncio.run(svc.capture_running(project, "manual")) + assert entry.path.exists() and entry.size > 0 + names = set(zipfile.ZipFile(entry.path).namelist()) + assert "data.db" in names, ( + f"PB archive missing data.db: {sorted(names)[:8]}" + ) + assert not list((pb_data / "backups").glob("*.zip")), ( + "archive must be MOVED out of pb_data/backups" + ) + + # bad credentials fail loudly, never a silent raw-copy fallback + (proj_dir / ".superuser").write_text( + _json.dumps({"email": email, "password": "wrong"}) + "\n" + ) + try: + asyncio.run(svc.capture_running(project, "manual")) + raise AssertionError("bad creds must raise") + except RuntimeError as e: + assert "auth failed" in str(e) + finally: + proc.terminate() + try: + proc.wait(timeout=5) + except Exception: + proc.kill() + print("§8 capture_running vs real PocketBase: OK") + +print("\nBackup acceptance (Phases 1-4): ALL GREEN") diff --git a/app/living_ui/test_data_safety.py b/app/living_ui/test_data_safety.py index 93f6be58..a10c0398 100644 --- a/app/living_ui/test_data_safety.py +++ b/app/living_ui/test_data_safety.py @@ -1,5 +1,14 @@ -"""Data-safety acceptance: baseline restore (build era) + staging copies -(modify era) keep agent/verifier test junk out of the production DB. +"""Data-safety acceptance for the unified dev/live lifecycle. + +The single invariant under test (docs/plans/living-ui-unified-lifecycle-plan.md): + + Nothing writes to a live environment's pb_data except PocketBase's + migration replay during Promoter.promote(). + +Every code change — first build or modify — develops and verifies in a DEV +environment (code copy, hidden port, FRESH schema-only DB); a clean verify +promotes. There is no stored "delivered" flag: first-vs-update is derived +from live_db_exists(). Run: python3 -m app.living_ui.test_data_safety @@ -32,7 +41,7 @@ from app.data.action import living_ui_actions as LA from app.living_ui.manager import LivingUIManager, LivingUIProject from app.living_ui.pb_data_io import restore_pb_data, snapshot_pb_data -from app.living_ui.staging import STAGING_PORT_RANGE, StagingSupervisor +from app.living_ui.lifecycle import DEV_PORT_RANGE, DevProvisioner, live_db_exists from app.living_ui.runner import LivingUIRunner from app.living_ui.wizard import _unwrap_document, adapt_chosen, fresh_build_chosen @@ -177,7 +186,7 @@ def _run_action(handler, input_data: dict) -> dict: print("§2 pb_data_io guards: OK") -# ── §3 StagingSupervisor ─────────────────────────────────────────────────── +# ── §3 DevProvisioner ────────────────────────────────────────────────────── class _StubRunner: @@ -190,43 +199,45 @@ async def kit_sync(self, project_dir): with tempfile.TemporaryDirectory() as tmp: living = Path(tmp) / "living_ui" - proj = _make_project_dir(living, "stage0001", 3125) - project = _Project("stage0001", proj, 3125) - # Trigger declaration MUST travel to staging: without it the copy's guard - # declares nothing, every ⚡ fire 400s, and the walker fails an + proj = _make_project_dir(living, "dev00001", 3125) + project = _Project("dev00001", proj, 3125) + # Trigger declaration MUST travel to the dev copy: without it the copy's + # guard declares nothing, every ⚡ fire 400s, and the walker fails an # unfixable "defect" (observed live 2026-08-06 — three identical STUCKs). (proj / "triggers.json").write_text( '{"triggers": {"ping": {"instruction": "reply", "description": "d"}}}' ) runner = _StubRunner() - sup = StagingSupervisor(living, runner) + sup = DevProvisioner(living, runner) inst = asyncio.run(sup.create_copy(project)) sdir = inst.dir - assert sdir == living / "_staging" / "project" / "stage0001" - assert STAGING_PORT_RANGE[0] <= inst.port <= STAGING_PORT_RANGE[1] + assert sdir == living / "_staging" / "project" / "dev00001" + assert DEV_PORT_RANGE[0] <= inst.port <= DEV_PORT_RANGE[1] manifest = _json.loads((sdir / "manifest.json").read_text()) assert manifest["port"] == inst.port, "manifest.port must be rewritten" + assert manifest["env"] == "dev", "dev copies must be stamped env=dev (A2APP)" assert str(inst.port) in manifest["pipeline"]["start"], "pipeline keeps port inline" assert "3125" not in manifest["pipeline"]["start"], "old port must be gone" assert runner.kit_synced == [sdir], "hash canon must be re-recorded after rewrite" assert not (sdir / "pb" / "pb_public").exists(), ( "gate rebuilds pb_public — never copy" ) + # THE POINT of the unified lifecycle: the dev copy has NO database at + # all — PocketBase creates it at boot and replays the migration chain. + # Live data is never cloned into an environment the agent writes to. + assert not (sdir / "pb" / "pb_data").exists(), ( + "dev copy must NOT contain a database — schema comes from migrations" + ) assert (sdir / "frontend" / "node_modules" / "somepkg").exists(), ( "node_modules rides along" ) assert (sdir / ".superuser").exists() and (sdir / ".lui").exists() assert (sdir / "triggers.json").exists(), ( - "triggers.json must travel to staging — its absence 400s every fire" + "triggers.json must travel to the dev copy — its absence 400s every fire" ) - # DB isolation: staging writes never reach the original. - assert _count(sdir / "pb" / "pb_data" / "data.db") == 2 - _mkdb(sdir / "pb" / "pb_data" / "data.db", rows=9) - assert _count(proj / "pb" / "pb_data" / "data.db") == 2, "original DB polluted!" - - # sync_code: refreshes agent-owned paths, keeps staging pb_data + manifest. + # sync_code: refreshes agent-owned paths, keeps the rewritten manifest. (proj / "frontend" / "src" / "App.tsx").write_text("export const A = 2\n") (proj / "frontend" / "package.json").write_text( '{"name": "app", "dependencies": {"x": "1.0.0"}}' @@ -237,34 +248,38 @@ async def kit_sync(self, project_dir): sup.sync_code(project, sdir) assert "A = 2" in (sdir / "frontend" / "src" / "App.tsx").read_text() assert "pong" in (sdir / "triggers.json").read_text(), ( - "fix-iteration edits to triggers.json must reach staging" + "fix-iteration edits to triggers.json must reach the dev copy" ) assert not (sdir / "frontend" / "node_modules").exists(), ( "changed package.json must clear node_modules so install runs" ) - assert _count(sdir / "pb" / "pb_data" / "data.db") == 11, ( - "sync_code must not touch staging data" - ) assert _json.loads((sdir / "manifest.json").read_text())["port"] == inst.port + # reset_db drops the dev DB (a booted dev instance leaves one behind); + # the next boot replays migrations from empty. + _mkdb(sdir / "pb" / "pb_data" / "data.db", rows=9) + sup.reset_db(sdir) + assert not (sdir / "pb" / "pb_data").exists(), "reset_db must drop the dev DB" + assert _count(proj / "pb" / "pb_data" / "data.db") == 2, "original DB polluted!" + # guarded rmtree refuses anything outside _staging try: sup._guarded_rmtree(proj) - raise AssertionError("guarded rmtree left the staging root!") + raise AssertionError("guarded rmtree left the dev root!") except ValueError: pass # destroy + reap - sup.destroy("stage0001", inst.to_record()) + sup.destroy("dev00001", inst.to_record()) assert not sdir.exists() leftover = living / "_staging" / "project" / "leftover99" leftover.mkdir(parents=True) reaped = sup.reap_all({"gone12345": {"dir": str(leftover), "pid": 99999999}}) assert reaped >= 1 and not leftover.exists() -print("§3 StagingSupervisor: OK") +print("§3 DevProvisioner: OK") -# ── §4 FactoryHost delivery helpers ──────────────────────────────────────── +# ── §4 FactoryHost delivery bookkeeping + live_db_exists ─────────────────── with tempfile.TemporaryDirectory() as tmp: living = Path(tmp) / "living_ui" @@ -277,22 +292,32 @@ def get_project(self, pid): living_ui_mod.get_living_ui_manager = lambda: _MgrOne() host = host_mod.FactoryHost() - assert host.is_delivered("sidecar01") is False - host.mark_delivered("sidecar01") - assert host.is_delivered("sidecar01") is True + # delivered_at is a cosmetic stamp, written once, never a control input. + assert host.delivered_at("sidecar01") is None + host.stamp_delivered("sidecar01") + first_stamp = host.delivered_at("sidecar01") + assert first_stamp is not None + host.stamp_delivered("sidecar01") + assert host.delivered_at("sidecar01") == first_stamp, "stamp is write-once" assert host.get_staging_record("sidecar01") is None host.set_staging_record("sidecar01", {"url": "http://127.0.0.1:3901", "port": 3901}) assert host.get_staging_record("sidecar01")["port"] == 3901 - # delivered flag survives alongside the staging record + # stamp survives alongside the dev record side = _json.loads((proj / ".factory" / "host.json").read_text()) - assert side["delivered"] is True and side["staging"]["port"] == 3901 + assert side["delivered_at"] == first_stamp and side["staging"]["port"] == 3901 host.clear_staging_record("sidecar01") assert host.get_staging_record("sidecar01") is None - assert host.is_delivered("sidecar01") is True -print("§4 FactoryHost delivery helpers: OK") + + # live_db_exists: the structural first-vs-update predicate. + assert live_db_exists(proj) is True + (proj / "pb" / "pb_data" / "data.db").unlink() + assert live_db_exists(proj) is False + assert live_db_exists("") is False and live_db_exists(None) is False + _mkdb(proj / "pb" / "pb_data" / "data.db", rows=2) # restore for reuse +print("§4 FactoryHost bookkeeping + live_db_exists: OK") -# ── §5 manager.launch_staging / finalize_modify / finalize_first_delivery ── +# ── §5 manager.open_dev / promote — THE INVARIANT ────────────────────────── with tempfile.TemporaryDirectory() as tmp: workspace = Path(tmp) @@ -309,12 +334,11 @@ def get_project(self, pid): ) project.bridge_token = "tok" mgr.projects["mgrtest01"] = project - mgr.staging.runner = _StubRunner() # no node in tests + mgr.lifecycle.provisioner.runner = _StubRunner() # no node in tests living_ui_mod.get_living_ui_manager = lambda: mgr host_mod._HOST = None # fresh singleton bound to this manager host = host_mod.get_factory_host() - host.mark_delivered("mgrtest01") PIPELINE_RUNS = [] @@ -338,9 +362,17 @@ async def _fake_pipeline(project_dir, port, bridge_token): return {"status": "success", "process": _FakeProc()} mgr._run_launch_pipeline = _fake_pipeline + mgr.lifecycle._launch_pipeline = _fake_pipeline + + # Fingerprint the LIVE DB before the whole arc: the invariant is that + # no lifecycle step below changes a single byte of it (the fake launch + # stands in for the promote boot, whose migration replay is the one + # sanctioned writer). + _live_db = proj_dir / "pb" / "pb_data" / "data.db" + _live_bytes_before = _live_db.read_bytes() - result = asyncio.run(mgr.launch_staging("mgrtest01")) - assert result["status"] == "success" and result.get("staging") is True + result = asyncio.run(mgr.open_dev("mgrtest01")) + assert result["status"] == "success" and result.get("dev") is True record = host.get_staging_record("mgrtest01") assert record and record["pid"] == 4242 sdir = Path(record["dir"]) @@ -348,79 +380,85 @@ async def _fake_pipeline(project_dir, port, bridge_token): "pipeline must target the COPY" ) assert PIPELINE_RUNS[-1][1] == record["port"] != 3127 + assert result.get("dir") == str(sdir), "agents need the dev dir for logs/CLI" + assert not (sdir / "pb" / "pb_data").exists(), "dev copy must start with no DB" + # live DB exists → open_dev re-armed the machine as a MODIFY + machine = host.machine_for("mgrtest01") + assert machine is not None and machine.state == "modifying", machine.state + + # a second open_dev (fix iteration) reuses the copy and resets its DB + _mkdb(sdir / "pb" / "pb_data" / "data.db", rows=9) # simulated boot junk + result = asyncio.run(mgr.open_dev("mgrtest01")) + assert result["status"] == "success" + assert not (sdir / "pb" / "pb_data").exists(), ( + "each open_dev must reset the dev DB — migrations replay from empty" + ) - # flip: relaunch real app, then destroy staging + record - FLIPPED = [] + # promote (update): relaunch real app, then destroy dev copy + record + PROMOTED = [] async def _fake_launch_and_verify(pid): - FLIPPED.append(pid) + PROMOTED.append(pid) return {"status": "success", "url": "http://127.0.0.1:3127", "port": 3127} - mgr.launch_and_verify = _fake_launch_and_verify - flip = asyncio.run(mgr.finalize_modify("mgrtest01")) - assert flip["status"] == "success" and FLIPPED == ["mgrtest01"] - assert not sdir.exists(), "flip must destroy the staging copy" + mgr.lifecycle.promoter._launch_live = _fake_launch_and_verify + up = asyncio.run(mgr.promote("mgrtest01")) + assert up["status"] == "success" and PROMOTED == ["mgrtest01"] + assert up["first"] is False, "live DB existed — this is an UPDATE promote" + assert not sdir.exists(), "promote must destroy the dev copy" assert host.get_staging_record("mgrtest01") is None + assert host.delivered_at("mgrtest01") is not None, "promote stamps delivery" + assert _live_db.read_bytes() == _live_bytes_before, ( + "INVARIANT VIOLATED: the live DB changed outside the promote boot" + ) - # failed flip keeps the copy and the record - result = asyncio.run(mgr.launch_staging("mgrtest01")) + # failed promote keeps the copy and the record + result = asyncio.run(mgr.open_dev("mgrtest01")) sdir = Path(host.get_staging_record("mgrtest01")["dir"]) async def _failing_launch(pid): return {"status": "error", "step": "health", "errors": ["boom"]} - mgr.launch_and_verify = _failing_launch - flip = asyncio.run(mgr.finalize_modify("mgrtest01")) - assert flip["status"] == "error" + mgr.lifecycle.promoter._launch_live = _failing_launch + up = asyncio.run(mgr.promote("mgrtest01")) + assert up["status"] == "error" assert sdir.exists() and host.get_staging_record("mgrtest01") is not None + mgr.lifecycle.provisioner.destroy("mgrtest01", host.get_staging_record("mgrtest01")) + host.clear_staging_record("mgrtest01") - # finalize_first_delivery: junk after baseline → restored before announce + # FIRST promote: no live DB → first=True; nothing restores or wipes. proj2_dir = _make_project_dir(living, "firstdel01", 3128) + import shutil as _shutil + + _shutil.rmtree(proj2_dir / "pb" / "pb_data") # a never-delivered build project2 = LivingUIProject( id="firstdel01", name="firstdel01", description="t", path=str(proj2_dir), - status="running", + status="stopped", port=3128, ) project2.bridge_token = "tok" mgr.projects["firstdel01"] = project2 - snapshot_pb_data( - proj2_dir / "pb" / "pb_data", proj2_dir / ".snapshots" / "baseline", living - ) - _mkdb(proj2_dir / "pb" / "pb_data" / "data.db", rows=6) # verifier junk - async def _fake_start(project_dir, port, bridge_token=""): - return _FakeProc() + dev = asyncio.run(mgr.open_dev("firstdel01")) + assert dev["status"] == "success" + # no live DB → build era: the machine must NOT be re-armed into modify + m2 = host.machine_for("firstdel01") + assert m2 is not None and m2.state == "building", m2.state - async def _fake_healthy(port, timeout=None): - return True - - mgr.runner.start = _fake_start - mgr.runner.wait_healthy = _fake_healthy - fin = asyncio.run(mgr.finalize_first_delivery("firstdel01")) - assert fin["status"] == "success" and fin["restored"] is True - assert _count(proj2_dir / "pb" / "pb_data" / "data.db") == 2, ( - "junk survived delivery!" - ) - assert project2.status == "running" + async def _first_launch(pid): + # the promote boot creates the live DB from migrations — simulate it + _mkdb(proj2_dir / "pb" / "pb_data" / "data.db", rows=0) + return {"status": "success", "url": "http://127.0.0.1:3128", "port": 3128} - # no baseline → deliver as-is, never guess-wipe - proj3_dir = _make_project_dir(living, "legacy0001", 3129) - project3 = LivingUIProject( - id="legacy0001", - name="legacy0001", - description="t", - path=str(proj3_dir), - status="running", - port=3129, - ) - mgr.projects["legacy0001"] = project3 - fin = asyncio.run(mgr.finalize_first_delivery("legacy0001")) - assert fin["status"] == "success" and fin["restored"] is False - assert _count(proj3_dir / "pb" / "pb_data" / "data.db") == 2 -print("§5 manager staging/finalize: OK") + mgr.lifecycle.promoter._launch_live = _first_launch + up = asyncio.run(mgr.promote("firstdel01")) + assert up["status"] == "success" and up["first"] is True + assert live_db_exists(proj2_dir) + assert host.get_staging_record("firstdel01") is None +print("§5 manager open_dev/promote invariant: OK") # ── §6-8 action branching (bare-exec, like the real executor) ────────────── @@ -432,6 +470,7 @@ async def _fake_healthy(port, timeout=None): class _StubMgr: def __init__(self, project): self._p = project + self.projects = {project.id: project} def get_project(self, pid): return self._p if pid == self._p.id else None @@ -440,26 +479,28 @@ async def launch_and_verify(self, pid): EVENTS.append("launch_and_verify") return {"status": "success", "url": self._p.url, "port": self._p.port} - async def launch_staging(self, pid): - EVENTS.append("launch_staging") + async def open_dev(self, pid): + EVENTS.append("open_dev") return { "status": "success", "url": "http://127.0.0.1:3901", "port": 3901, - "staging": True, + "dir": "/tmp/devcopy", + "dev": True, } async def stop_project(self, pid): EVENTS.append("stop_project") return True - async def finalize_first_delivery(self, pid): - EVENTS.append("finalize_first_delivery") - return {"status": "success", "restored": True} - - async def finalize_modify(self, pid): - EVENTS.append("finalize_modify") - return {"status": "success"} + async def promote(self, pid): + EVENTS.append("promote") + return { + "status": "success", + "url": self._p.url, + "port": self._p.port, + "first": False, + } async def _b_ready(pid, url, port): @@ -494,51 +535,27 @@ def _wire(project, host): with tempfile.TemporaryDirectory() as tmp: living = Path(tmp) / "living_ui" - # §6a build mode, clean verdict: finalize + mark_delivered BEFORE announce - proj = _make_project_dir(living, "actbuild01", 3131) - project = _Project("actbuild01", proj, 3131) - host_mod._HOST = None - host = host_mod.get_factory_host() - _wire(project, host) - EVENTS.clear() - WALK["report"] = { - "kind": "pass", - "passed": ["feature one"], - "defects": [], - "raw": "VERDICT: PASS", - } - out = _run_action(LA.living_ui_walk_verify, {"project_id": "actbuild01"}) - assert out["status"] == "success", out - assert WALK["base_url"] == "http://127.0.0.1:3131" and WALK["project_path"] is None - fin_i = EVENTS.index("finalize_first_delivery") - ready_i = next( - i - for i, e in enumerate(EVENTS) - if isinstance(e, tuple) and e[0] == "broadcast_ready" - ) - assert fin_i < ready_i, "restore must precede the delivery announce" - assert host.is_delivered("actbuild01") is True - print("§6a build clean → finalize→mark→announce: OK") - - # §6b delivered but no staging: walk refuses, notify_ready boots staging + # §6a no dev env: walk refuses; notify_ready boots the dev env proj = _make_project_dir(living, "actnostg01", 3132) project = _Project("actnostg01", proj, 3132) + host_mod._HOST = None + host = host_mod.get_factory_host() stub = _wire(project, host) - host.mark_delivered("actnostg01") EVENTS.clear() out = _run_action(LA.living_ui_walk_verify, {"project_id": "actnostg01"}) - assert out["status"] == "error" and "staging" in out["message"], out + assert out["status"] == "error" and "dev" in out["message"].lower(), out out = _run_action(LA.living_ui_notify_ready, {"project_id": "actnostg01"}) - assert out["status"] == "success" and "launch_staging" in EVENTS - assert "launch_and_verify" not in EVENTS - assert "STAGING" in out["message"] - print("§6b delivered gating: OK") + assert out["status"] == "success" and "open_dev" in EVENTS + assert "launch_and_verify" not in EVENTS, "native apps never launch live here" + assert "DEV environment" in out["message"] + assert "/tmp/devcopy" in out["message"], "message must name the dev dir" + print("§6a dev-env gating: OK") - # §6c staging defects: live app NOT stopped; staging log quoted + # §6c dev defects: live app NOT stopped; dev log quoted sdir = living / "_staging" / "project" / "actnostg01" (sdir / "logs").mkdir(parents=True) (sdir / "logs" / "pocketbase.log").write_text( - "ERROR hook exploded: staging-only-line\n" + "ERROR hook exploded: dev-only-line\n" ) host.set_staging_record( "actnostg01", {"url": "http://127.0.0.1:3905", "port": 3905, "dir": str(sdir)} @@ -556,16 +573,16 @@ def _wire(project, host): } out = _run_action(LA.living_ui_walk_verify, {"project_id": "actnostg01"}) assert out["status"] == "error" - assert "stop_project" not in EVENTS, "modify defects must not stop the live app" + assert "stop_project" not in EVENTS, "native defects must not stop the live app" assert "previous working version" in out["message"] assert WALK["base_url"] == "http://127.0.0.1:3905", "verifier must drive the COPY" assert WALK["project_path"] == str(sdir) - assert "staging-only-line" in captured.get("server_log", ""), ( - "evidence must come from the staging log" + assert "dev-only-line" in captured.get("server_log", ""), ( + "evidence must come from the dev log" ) - print("§6c staging defects: OK") + print("§6c dev defects: OK") - # §6d staging clean: flip before announce; flip failure blocks announce + # §6d clean verdict: promote before announce; promote failure blocks it host.report_verify = lambda *a, **k: types.SimpleNamespace( next_state="done", payload={} ) @@ -578,46 +595,54 @@ def _wire(project, host): } out = _run_action(LA.living_ui_walk_verify, {"project_id": "actnostg01"}) assert out["status"] == "success", out - flip_i = EVENTS.index("finalize_modify") + promote_i = EVENTS.index("promote") ready_i = next( i for i, e in enumerate(EVENTS) if isinstance(e, tuple) and e[0] == "broadcast_ready" ) - assert flip_i < ready_i, "deploy must precede the announce" + assert promote_i < ready_i, "the promote must precede the announce" assert EVENTS[ready_i][1] == "http://127.0.0.1:3132", ( "announce must carry the REAL url" ) + assert "finalize_first_delivery" not in EVENTS, ( + "the baseline-restore path must not exist" + ) - class _FlipFailMgr(_StubMgr): - async def finalize_modify(self, pid): - EVENTS.append("finalize_modify") + class _PromoteFailMgr(_StubMgr): + async def promote(self, pid): + EVENTS.append("promote") return { "status": "error", "step": "health", "errors": ["real app did not boot"], } - living_ui_mod.get_living_ui_manager = lambda: _FlipFailMgr(project) + living_ui_mod.get_living_ui_manager = lambda: _PromoteFailMgr(project) EVENTS.clear() out = _run_action(LA.living_ui_walk_verify, {"project_id": "actnostg01"}) assert out["status"] == "error" and "deploy" in out["message"].lower() assert not any( isinstance(e, tuple) and e[0] == "broadcast_ready" for e in EVENTS - ), "a failed deploy must never announce" - print("§6d staging clean/flip: OK") + ), "a failed promote must never announce" + print("§6d clean verdict promotes: OK") - # §7 build mode notify_ready unchanged + # §7 notify_ready always routes native apps to the dev env — even a + # first build with no live DB (the unified flow's whole point). proj = _make_project_dir(living, "actnr0001", 3133) + import shutil as _sh + + _sh.rmtree(proj / "pb" / "pb_data") # a never-delivered scaffold project = _Project("actnr0001", proj, 3133) _wire(project, host) EVENTS.clear() out = _run_action(LA.living_ui_notify_ready, {"project_id": "actnr0001"}) - assert out["status"] == "success" and "launch_and_verify" in EVENTS - assert "STAGING" not in out["message"] - print("§7 notify_ready build mode: OK") + assert out["status"] == "success" and "open_dev" in EVENTS + assert "launch_and_verify" not in EVENTS + assert "DEV environment" in out["message"] + print("§7 notify_ready first build → dev env: OK") - # §8 living_ui_http: staging redirect, no iframe reload, write refusal + # §8 living_ui_http: dev redirect, no iframe reload, mid-arc refusal import requests as _requests _orig_request = _requests.request @@ -640,31 +665,21 @@ def _fake_request(method, url, **kwargs): project = _Project("acthttp01", proj, 3134) _wire(project, host) - # not delivered → real app + data_changed dispatch - EVENTS.clear() - out = _run_action( - LA.living_ui_http, - {"project_id": "acthttp01", "method": "POST", "path": "/api/x", "json": {}}, - ) - assert out["status"] == "success" and HTTP[-1][1].startswith( - "http://127.0.0.1:3134" - ) - assert "data_changed" in EVENTS - - # delivered, no staging → writes refused, reads allowed - host.mark_delivered("acthttp01") + # mid-arc (machine non-terminal), no dev env → writes refused, + # reads allowed. A virgin machine reads as mid-arc — that is the + # safe direction: agent test writes belong in the dev env. out = _run_action( LA.living_ui_http, {"project_id": "acthttp01", "method": "POST", "path": "/api/x", "json": {}}, ) - assert out["status"] == "error" and "staging" in out["message"], out + assert out["status"] == "error" and "dev" in out["message"], out out = _run_action( LA.living_ui_http, {"project_id": "acthttp01", "method": "GET", "path": "/api/x"}, ) assert out["status"] == "success" - # delivered + staging → redirected, and NO iframe reload + # dev env up → ALL agent HTTP redirected there, and NO iframe reload host.set_staging_record( "acthttp01", {"url": "http://127.0.0.1:3906", "port": 3906, "dir": "x"} ) @@ -678,8 +693,23 @@ def _fake_request(method, url, **kwargs): "write must hit the COPY" ) assert "data_changed" not in EVENTS, ( - "staging writes must not reload the user's iframe" + "dev writes must not reload the user's iframe" ) + + # arc closed (machine terminal), no dev env → live writes are USER + # data and flow to the real app + data_changed dispatch. + host.clear_staging_record("acthttp01") + host._machines["acthttp01"] = types.SimpleNamespace(terminal=True) + EVENTS.clear() + out = _run_action( + LA.living_ui_http, + {"project_id": "acthttp01", "method": "POST", "path": "/api/x", "json": {}}, + ) + assert out["status"] == "success" and HTTP[-1][1].startswith( + "http://127.0.0.1:3134" + ) + assert "data_changed" in EVENTS + host._machines.pop("acthttp01", None) finally: _requests.request = _orig_request print("§8 living_ui_http redirect/refusal: OK") @@ -906,6 +936,10 @@ async def _fake_fu_llm(system_prompt, user_prompt, prompt_name): with tempfile.TemporaryDirectory() as tmp: living = Path(tmp) / "living_ui" proj = _make_project_dir(living, "adopt0001", 3141) + import shutil as _sh11 + + # A wizard scaffold mid-build has NO live DB (builds run in the dev env). + _sh11.rmtree(proj / "pb" / "pb_data") project = _Project("adopt0001", proj, 3141) host_mod._HOST = None host = host_mod.get_factory_host() @@ -956,10 +990,10 @@ async def _b_created(p): assert INSTALLS[-1] == "adopt0001" and not VERDICTS assert "notify_ready" in out["message"] and "adaptations" in out["message"].lower() - # c) delivered session project holding the SAME app → idempotent no-op + # c) session project holding the SAME installed app → idempotent no-op # (the crash-resume path: redispatched "continue build" must not mint a - # duplicate) - host.mark_delivered("adopt0001") + # duplicate). "Installed" is structural: live DB + marketplaceAppId. + _mkdb(proj / "pb" / "pb_data" / "data.db", rows=2) _mf = _json.loads((proj / "manifest.json").read_text()) _mf["marketplaceAppId"] = "kanban-board" (proj / "manifest.json").write_text(_json.dumps(_mf)) @@ -1005,12 +1039,12 @@ async def _b_created(p): ) project.bridge_token = "tok" mgr.projects["modarc001"] = project - mgr.staging.runner = _StubRunner() + mgr.lifecycle.provisioner.runner = _StubRunner() living_ui_mod.get_living_ui_manager = lambda: mgr host_mod._HOST = None host = host_mod.get_factory_host() - host.mark_delivered("modarc001") + host.stamp_delivered("modarc001") assert isinstance(host.delivered_at("modarc001"), float) # Simulate the finished BUILD arc (wizard-built app): machine at DONE. @@ -1046,9 +1080,11 @@ async def _mod_pipeline(project_dir, port, bridge_token): return {"status": "success", "process": _ModProc()} mgr._run_launch_pipeline = _mod_pipeline + mgr.lifecycle._launch_pipeline = _mod_pipeline - # First modify: staging up → machine re-armed into MODIFYING, gen 1 - result = asyncio.run(mgr.launch_staging("modarc001")) + # First modify: dev env up (live DB exists) → machine re-armed into + # MODIFYING, gen 1 + result = asyncio.run(mgr.open_dev("modarc001")) assert result["status"] == "success" machine = host.machine_for("modarc001") assert machine.state == "modifying" and machine.generation == 1 @@ -1070,8 +1106,8 @@ async def _mod_pipeline(project_dir, port, bridge_token): assert machine.state == "fixing" assert MISSIONS and MISSIONS[-1][0] == "fix" and MISSIONS[-1][1] == 1 - # Fix mission re-enters launch_staging → begin_modify no-ops mid-arc - result = asyncio.run(mgr.launch_staging("modarc001")) + # Fix mission re-enters open_dev → begin_modify no-ops mid-arc + result = asyncio.run(mgr.open_dev("modarc001")) assert result["status"] == "success" assert machine.state == "fixing" and machine.generation == 1 @@ -1087,7 +1123,7 @@ async def _mod_pipeline(project_dir, port, bridge_token): assert CHAT and "change is live" in CHAT[-1], CHAT # Second modify: fresh generation, fresh budget - result = asyncio.run(mgr.launch_staging("modarc001")) + result = asyncio.run(mgr.open_dev("modarc001")) assert machine.state == "modifying" and machine.generation == 2 state_file = _json.loads((proj_dir / ".factory" / "state.json").read_text()) assert state_file["total_missions"] == 0 and len(state_file["generations"]) == 2 @@ -1102,7 +1138,7 @@ async def _mod_pipeline(project_dir, port, bridge_token): host_mod._HOST = None host = host_mod.get_factory_host() stub = _wire(project, host) - host.mark_delivered("specbelt01") + host.stamp_delivered("specbelt01") host.set_staging_record( "specbelt01", {"url": "http://127.0.0.1:3907", "port": 3907, "dir": "x"} ) @@ -1164,7 +1200,9 @@ async def _mod_pipeline(project_dir, port, bridge_token): _mf = _json.loads((dest / "manifest.json").read_text()) assert _mf["id"] == project.id and _mf["port"] == project.port assert str(project.port) in _mf["pipeline"]["start"] - assert host.is_delivered(project.id), "imports are delivered on arrival" + assert host.delivered_at(project.id) is not None, ( + "imports are stamped delivered on arrival" + ) assert (src_dir / ".superuser").exists(), "the source folder is never modified" # zip import through the same core @@ -1176,7 +1214,7 @@ async def _mod_pipeline(project_dir, port, bridge_token): if f.is_file() and ".git" not in f.parts and "node_modules" not in f.parts: zf.write(f, Path("exported_app") / f.relative_to(src_dir)) project2 = asyncio.run(mgr.import_project_source(str(zip_path))) - assert project2.id != project.id and host.is_delivered(project2.id) + assert project2.id != project.id and host.delivered_at(project2.id) is not None # git import via a real local repo (file:// clone path) import subprocess as _sub @@ -1194,7 +1232,7 @@ async def _mod_pipeline(project_dir, port, bridge_token): ): _sub.run(cmd, cwd=git_src, check=True, capture_output=True) project3 = asyncio.run(mgr.import_project_source(f"file://{git_src}")) - assert host.is_delivered(project3.id) + assert host.delivered_at(project3.id) is not None assert len({project.id, project2.id, project3.id}) == 3 # A TEMPLATE tree (marketplace checkout imported by path) must have its @@ -1358,7 +1396,7 @@ async def _fake_source_llm(system_prompt, user_prompt, prompt_name): req_text = (dest / "reference" / "requirements.md").read_text() assert "The user can add a todo." in req_text assert "## Original source" in req_text and "reference/source/" in req_text - assert not host.is_delivered(project.id), "a conversion is a pre-delivery BUILD" + assert host.delivered_at(project.id) is None, "a conversion is a pre-delivery BUILD" # A native Living UI source must be refused toward living_ui_import v2src = Path(tmp) / "v2app" @@ -1431,7 +1469,7 @@ async def _sdr2(pid, **kwargs): # foreign folder → EXTERNAL registration (not delivered, craftbot.json) project = asyncio.run(mgr.import_project_source(str(site), name="Ext Site")) assert project.project_type == "external" and project.app_runtime == "static" - assert project.status == "stopped" and not host.is_delivered(project.id) + assert project.status == "stopped" and host.delivered_at(project.id) is None cfg = _json.loads((Path(project.path) / "craftbot.json").read_text()) assert cfg["external"] is True and cfg["port"] == project.port assert cfg["pipeline"]["start"] == "", "adoption fills the verbs" @@ -1466,10 +1504,9 @@ async def _sdr2(pid, **kwargs): finally: asyncio.run(mgr.stop_project(project.id)) - # launch_staging refuses externals (changes run live) - host.mark_delivered(project.id) - res = asyncio.run(mgr.launch_staging(project.id)) - assert res["status"] == "error" and "no staging" in res["errors"][0] + # open_dev refuses externals (changes run live) + res = asyncio.run(mgr.open_dev(project.id)) + assert res["status"] == "error" and "no dev environment" in res["errors"][0] # broken start command → health failure with app.log evidence cfg["pipeline"]["start"] = "python3 -c 'import sys; sys.exit(3)'" @@ -1479,7 +1516,7 @@ async def _sdr2(pid, **kwargs): print("§19 external apps run as-is: OK") -# ── §20 delivered EXTERNAL app skips staging in the actions ──────────────── +# ── §20 EXTERNAL apps skip the dev env in the actions ────────────────────── with tempfile.TemporaryDirectory() as tmp: living = Path(tmp) / "living_ui" proj = _make_project_dir(living, "extact0001", 3154) @@ -1489,17 +1526,17 @@ async def _sdr2(pid, **kwargs): host_mod._HOST = None host = host_mod.get_factory_host() stub = _wire(project, host) - host.mark_delivered("extact0001") EVENTS.clear() out = _run_action(LA.living_ui_notify_ready, {"project_id": "extact0001"}) assert out["status"] == "success" - assert "launch_and_verify" in EVENTS and "launch_staging" not in EVENTS, ( - "delivered externals must relaunch LIVE, never stage" + assert "launch_and_verify" in EVENTS and "open_dev" not in EVENTS, ( + "externals must relaunch LIVE, never open a dev env" ) assert "EXTERNAL app runs live" in out["message"] - # walk_verify: no staging requirement; build-mode branches apply + # walk_verify: no dev-env requirement; a clean verdict promotes + # (bookkeeping only for externals — the new code already runs live) EVENTS.clear() WALK["report"] = { "kind": "pass", @@ -1509,11 +1546,8 @@ async def _sdr2(pid, **kwargs): } out = _run_action(LA.living_ui_walk_verify, {"project_id": "extact0001"}) assert out["status"] == "success", out - assert "finalize_first_delivery" in EVENTS, ( - "external clean verdict follows the (no-op-safe) build finalize" - ) - assert "finalize_modify" not in EVENTS -print("§20 delivered external action branches: OK") + assert "promote" in EVENTS, "external clean verdict still promotes (bookkeeping)" +print("§20 external action branches: OK") # ── §21 surrender loops are capped by the machine (chili3d incident) ─────── @@ -1574,7 +1608,7 @@ def get_project(self, pid): _mf["craftbotVersion"] = "0.9.9" # the original creator (src_dir / "manifest.json").write_text(_json.dumps(_mf)) # donor lifecycle state must NOT travel with an import (a fresh sidecar - # IS created by the import's own mark_delivered — check donor CONTENT) + # IS created by the import's own stamp_delivered — check donor CONTENT) (src_dir / ".factory").mkdir() (src_dir / ".factory" / "host.json").write_text( '{"delivered": true, "donor_marker": 1}' diff --git a/app/living_ui/test_trigger_plane.py b/app/living_ui/test_trigger_plane.py index 196c55e3..0fff383e 100644 --- a/app/living_ui/test_trigger_plane.py +++ b/app/living_ui/test_trigger_plane.py @@ -173,22 +173,18 @@ def _fire(bridge, token="good", trigger="restock_needed", request_id="row1"): assert _fire(bridge).status == 403 assert len(mgr.consent_asks) == 2, "a SUCCESSFUL ask must be hourly-capped" - # Consented but NOT delivered: build-era fires are verifier traffic. + # Consented + a DEV environment active: fires are agent/verifier test + # traffic (the walker clicks ⚡ in the dev instance, which aliases to the + # real project id through the shared bridge token) — must defer. host.set_triggers_approved("gates001") - resp = _fire(bridge) - assert resp.status == 200 and b"deferred" in resp.body, ( - "pre-delivery fire must defer, not dispatch" - ) - assert mgr.notified == [] - - # Delivered + staging copy active: modify-era fires must also defer. - host.mark_delivered("gates001") host.set_staging_record("gates001", {"dir": "/tmp/x", "port": 3901, "pid": 1}) resp = _fire(bridge) - assert resp.status == 200 and b"deferred" in resp.body, "staging fire must defer" + assert resp.status == 200 and b"deferred" in resp.body, "dev-env fire must defer" assert mgr.notified == [] - # Live era: delivered, no staging → dispatch exactly once. + # Live era: consented, no dev env → dispatch exactly once. (No stored + # "delivered" flag any more — with no dev env in flight, a consented + # fire from a running app is legitimate operation.) host.clear_staging_record("gates001") resp = _fire(bridge) assert resp.status == 200 and b"deferred" not in resp.body diff --git a/app/living_ui/walk_verify.py b/app/living_ui/walk_verify.py index 5e172e1c..ad79d315 100644 --- a/app/living_ui/walk_verify.py +++ b/app/living_ui/walk_verify.py @@ -35,7 +35,7 @@ async def run_walk_verify( """Run the walk_verify sub-agent for a running project. base_url/project_path override where the verifier drives and reads — - used by staging mode on delivered apps, where the app under test is a + used to point it at the DEV environment, where the app under test is a disposable copy on a hidden port, never the user's live instance. Defaults preserve the original behavior (the registered project). diff --git a/app/ui_layer/adapters/browser_adapter.py b/app/ui_layer/adapters/browser_adapter.py index 7a7c8834..4f84d9ea 100644 --- a/app/ui_layer/adapters/browser_adapter.py +++ b/app/ui_layer/adapters/browser_adapter.py @@ -1658,6 +1658,24 @@ async def _handle_ws_message(self, data: Dict[str, Any], ws=None) -> None: project_id, setting, value ) + elif msg_type == "living_ui_backups_list": + await self._handle_living_ui_backups_list(data.get("projectId", "")) + + elif msg_type == "living_ui_backup_now": + await self._handle_living_ui_backup_now(data.get("projectId", "")) + + elif msg_type == "living_ui_backup_restore": + await self._handle_living_ui_backup_restore( + data.get("projectId", ""), data.get("filename", "") + ) + + elif msg_type == "living_ui_backup_delete": + await self._handle_living_ui_backup_delete( + data.get("projectId", ""), + data.get("filename", ""), + orphan=bool(data.get("orphan", False)), + ) + elif msg_type == "living_ui_marketplace_list": await self._handle_marketplace_list() @@ -1762,7 +1780,9 @@ async def _handle_ws_message(self, data: Dict[str, Any], ws=None) -> None: elif msg_type == "living_ui_delete": project_id = data.get("projectId", "") - await self._handle_living_ui_delete(project_id) + await self._handle_living_ui_delete( + project_id, delete_backups=bool(data.get("deleteBackups", False)) + ) elif msg_type == "living_ui_state_update": await self._handle_living_ui_state_update(data) @@ -3040,13 +3060,17 @@ async def _handle_living_ui_stop(self, project_id: str) -> None: } ) - async def _handle_living_ui_delete(self, project_id: str) -> None: + async def _handle_living_ui_delete( + self, project_id: str, delete_backups: bool = False + ) -> None: """Delete a Living UI project (and its dedicated session).""" try: project = self._living_ui_manager.get_project(project_id) session_id = project.session_id if project else None - success = await self._living_ui_manager.delete_project(project_id) + success = await self._living_ui_manager.delete_project( + project_id, delete_backups=delete_backups + ) try: from app.living_ui import construction_events @@ -6714,6 +6738,94 @@ async def _handle_living_ui_project_setting_update( {"type": "living_ui_project_setting_update", "data": result} ) + # Backups (spec docs/plans/living-ui-backups-plan.md Phase 4). Thin + # handlers: all policy lives in the manager/BackupStore. Restore and + # backup-now run as background tasks (stop+relaunch can take a minute) + # so the WS loop stays responsive; results broadcast with *_result types. + + async def _handle_living_ui_backups_list(self, project_id: str) -> None: + from app.living_ui import get_living_ui_manager + + payload = {"projectId": project_id, "backups": [], "totalSize": 0} + try: + manager = get_living_ui_manager() + entries = manager.backups.store.list_backups(project_id) + payload["backups"] = [ + { + "filename": e.filename, + "ts": int(e.ts * 1000), + "trigger": e.trigger, + "size": e.size, + } + for e in entries + ] + payload["totalSize"] = sum(e.size for e in entries) + except Exception as e: + payload["error"] = str(e) + await self._broadcast({"type": "living_ui_backups_list", "data": payload}) + + async def _handle_living_ui_backup_now(self, project_id: str) -> None: + from app.living_ui import get_living_ui_manager + + async def _run() -> None: + try: + result = await get_living_ui_manager().backup_now(project_id) + except Exception as e: + result = {"status": "error", "errors": [str(e)]} + await self._broadcast( + { + "type": "living_ui_backup_now_result", + "data": {"projectId": project_id, **result}, + } + ) + await self._handle_living_ui_backups_list(project_id) + + asyncio.create_task(_run()) + + async def _handle_living_ui_backup_restore( + self, project_id: str, filename: str + ) -> None: + from app.living_ui import get_living_ui_manager + + async def _run() -> None: + try: + result = await get_living_ui_manager().restore_backup( + project_id, filename + ) + except Exception as e: + result = {"status": "error", "errors": [str(e)]} + await self._broadcast( + { + "type": "living_ui_backup_restore_result", + "data": {"projectId": project_id, "filename": filename, **result}, + } + ) + await self._handle_living_ui_backups_list(project_id) + + asyncio.create_task(_run()) + + async def _handle_living_ui_backup_delete( + self, project_id: str, filename: str, orphan: bool = False + ) -> None: + from app.living_ui import get_living_ui_manager + + data = {"projectId": project_id, "filename": filename, "success": True} + try: + manager = get_living_ui_manager() + if orphan: + # Whole-dir cleanup of a deleted project's leftovers (D5) — + # refuse if the id is (again) a registered project. + if project_id in manager.projects: + raise ValueError("not an orphan — project exists") + manager.backups.store.delete_project_backups(project_id) + else: + manager.backups.store.delete(project_id, filename) + except Exception as e: + data = {**data, "success": False, "error": str(e)} + await self._broadcast({"type": "living_ui_backup_delete", "data": data}) + if not orphan: + await self._handle_living_ui_backups_list(project_id) + # ===================== # Playbook Handlers # ===================== diff --git a/app/ui_layer/browser/frontend/src/pages/Settings/LivingUISettings.tsx b/app/ui_layer/browser/frontend/src/pages/Settings/LivingUISettings.tsx index 89a40c7c..35e0fe24 100644 --- a/app/ui_layer/browser/frontend/src/pages/Settings/LivingUISettings.tsx +++ b/app/ui_layer/browser/frontend/src/pages/Settings/LivingUISettings.tsx @@ -8,6 +8,8 @@ import { Download, Copy, ChevronRight, + Archive, + RotateCcw, } from 'lucide-react' import { Button, ConfirmModal } from '../../components/ui' import { useConfirmModal } from '../../hooks' @@ -16,6 +18,7 @@ import { useSettingsWebSocket } from './useSettingsWebSocket' import { useAppDispatch, useAppSelector } from '../../store/hooks' import { updateProjectSetting, + setBackupBusy, type LivingUISettingsProject as LivingUIProject, } from '../../store/slices/livingUiSettingsSlice' import { @@ -88,9 +91,14 @@ export function LivingUISettings() { } const handleDelete = (project: LivingUIProject) => { + const backupCount = project.backupStatus?.count || 0 + const backupNote = + backupCount > 0 + ? ` Its ${backupCount} data backup${backupCount === 1 ? '' : 's'} will be KEPT and can be removed below afterwards.` + : '' confirm({ title: 'Delete Living UI', - message: `Are you sure you want to delete "${project.name}"? This will remove all project files and cannot be undone.`, + message: `Are you sure you want to delete "${project.name}"? This will remove all project files and cannot be undone.${backupNote}`, confirmText: 'Delete', variant: 'danger', }, () => { @@ -99,6 +107,19 @@ export function LivingUISettings() { }) } + const backupOrphans = useAppSelector(s => s.livingUiSettings.backupOrphans) + const handleDeleteOrphanBackups = (orphanId: string) => { + confirm({ + title: 'Delete leftover backups', + message: `Permanently delete all backup archives of the deleted app "${orphanId}"? They are the only remaining copy of its data.`, + confirmText: 'Delete backups', + variant: 'danger', + }, () => { + send('living_ui_backup_delete', { projectId: orphanId, filename: '', orphan: true }) + send('living_ui_settings_get') + }) + } + return (
@@ -135,11 +156,16 @@ export function LivingUISettings() { onStop={() => handleStop(project.id)} onDelete={() => handleDelete(project)} onToggleSetting={(setting, value) => { - // Optimistic so the toggle flips immediately; the refetch + // Optimistic so the control flips immediately; the refetch // triggered by the response reconciles authoritative state. dispatch(updateProjectSetting({ projectId: project.id, - setting: setting as 'autoLaunch' | 'logCleanup', + setting: setting as + | 'autoLaunch' + | 'logCleanup' + | 'backupsEnabled' + | 'backupInterval' + | 'backupKeep', value, })) send('living_ui_project_setting_update', { projectId: project.id, setting, value }) @@ -152,6 +178,54 @@ export function LivingUISettings() { )}
+ {/* ── Leftover backups of deleted apps (kept on delete — removable here) ── */} + {backupOrphans.length > 0 && ( +
+

Leftover backups

+

+ Backup archives of deleted apps. They are kept when an app is deleted; remove them here when you no longer need the data. +

+
+ {backupOrphans.map(orphanId => ( +
+ + + {orphanId} + +
+ ))} +
+
+ )} +
) @@ -168,7 +242,7 @@ interface ProjectCardProps { onLaunch: () => void onStop: () => void onDelete: () => void - onToggleSetting: (setting: string, value: boolean) => void + onToggleSetting: (setting: string, value: boolean | string | number) => void send: (type: string, data?: Record) => void onMessage: (type: string, handler: (data: unknown) => void) => () => void } @@ -493,6 +567,25 @@ function ProjectCard({ + {/* Zone 3b — Backups (native apps only: externals have no pb_data) */} + {project.projectType !== 'external' && ( +
+
+ Backups +
+ +
+ )} + {/* Zone 4 — Share */} {isRunning && (
= [ + { value: 'hourly', label: 'Every hour' }, + { value: '6h', label: 'Every 6 hours' }, + { value: 'daily', label: 'Daily' }, + { value: 'weekly', label: 'Weekly' }, +] + +const TRIGGER_LABELS: Record = { + scheduled: 'scheduled', + pre_promote: 'pre-update', + manual: 'manual', +} + +function fmtSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B` + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB` + return `${(bytes / (1024 * 1024)).toFixed(1)} MB` +} + +function fmtWhen(msEpoch: number): string { + return new Date(msEpoch).toLocaleString() +} + +interface BackupsSectionProps { + project: LivingUIProject + onToggleSetting: (setting: string, value: boolean | string | number) => void + send: (type: string, data?: Record) => void +} + +function BackupsSection({ project, onToggleSetting, send }: BackupsSectionProps) { + const dispatch = useAppDispatch() + const { modalProps: confirmModalProps, confirm } = useConfirmModal() + const backups = useAppSelector( + s => s.livingUiSettings.backupsByProject[project.id], + ) + const busy = useAppSelector( + s => s.livingUiSettings.backupBusy[project.id] || false, + ) + const status = project.backupStatus || {} + + // Fetch the archive list when the section first shows (card expanded). + useEffect(() => { + if (backups === undefined) + send('living_ui_backups_list', { projectId: project.id }) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [project.id, send]) + + const handleBackupNow = () => { + dispatch(setBackupBusy({ projectId: project.id, busy: true })) + send('living_ui_backup_now', { projectId: project.id }) + } + + const handleRestore = (filename: string, ts: number) => { + // Reversible by design (FR9): the backend captures the current state + // first and aborts if that fails — hence a plain consequence modal, + // not a typed confirmation. + confirm({ + title: 'Restore backup', + message: `Restore "${project.name}" to its state from ${fmtWhen(ts)}? Data created after that point will be removed — a backup of the current state is taken first, so this can be undone.`, + confirmText: 'Restore', + variant: 'danger', + }, () => { + dispatch(setBackupBusy({ projectId: project.id, busy: true })) + send('living_ui_backup_restore', { projectId: project.id, filename }) + }) + } + + const handleDeleteEntry = (filename: string, ts: number) => { + confirm({ + title: 'Delete backup', + message: `Permanently delete the backup from ${fmtWhen(ts)}?`, + confirmText: 'Delete', + variant: 'danger', + }, () => { + send('living_ui_backup_delete', { projectId: project.id, filename }) + }) + } + + const rowStyle: React.CSSProperties = { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + gap: 'var(--space-3)', + padding: 'var(--space-2) 0', + } + + return ( +
+ {/* Enable toggle */} +
+
+ Scheduled backups + + Back up this app's data and files automatically + +
+ onToggleSetting('backupsEnabled', e.target.checked)} + /> +
+ + {project.backupsEnabled && ( + <> +
+
+ Frequency +
+ +
+ +
+
+ Backups to keep + + Oldest scheduled backups are removed beyond this count + +
+ { + const v = parseInt(e.target.value, 10) + if (Number.isFinite(v) && v >= 1 && v <= 30) + onToggleSetting('backupKeep', v) + }} + style={{ + width: 64, + background: 'var(--bg-primary)', + color: 'var(--text-primary)', + border: '1px solid var(--border-primary)', + borderRadius: 'var(--radius-sm)', + padding: '4px 8px', + fontSize: 'var(--text-sm)', + }} + /> +
+ + )} + + {/* Status line + Back up now */} +
+ + {status.lastError + ? `Last backup failed: ${status.lastError}` + : status.lastAt + ? `Last backup ${fmtWhen(status.lastAt * 1000)} · ${status.count || 0} kept · ${fmtSize(status.totalSize || 0)}` + : 'No backups yet'} + + +
+ + {/* Archive list */} + {(backups || []).length > 0 && ( +
+ {(backups || []).map(b => ( +
+ + {fmtWhen(b.ts)} + + {' '}· {TRIGGER_LABELS[b.trigger] || b.trigger} · {fmtSize(b.size)} + + +
+ ))} +
+ )} + + +
+ ) +} + + // ── Share Section ────────────────────────────────────────────── interface ShareSectionProps { diff --git a/app/ui_layer/browser/frontend/src/store/slices/livingUiSettingsSlice.ts b/app/ui_layer/browser/frontend/src/store/slices/livingUiSettingsSlice.ts index 8d9e0c8a..40285661 100644 --- a/app/ui_layer/browser/frontend/src/store/slices/livingUiSettingsSlice.ts +++ b/app/ui_layer/browser/frontend/src/store/slices/livingUiSettingsSlice.ts @@ -4,6 +4,20 @@ import { register } from '../socket/messageRegistry' // Project shape used by the Settings > Living UI tab. Distinct from the // project shape used by `livingUiSlice` (which drives the main /living-ui // page) — this one carries the per-project preferences exposed in Settings. +export interface LivingUIBackupStatus { + lastAt?: number | null + lastError?: string | null + count?: number + totalSize?: number +} + +export interface LivingUIBackupEntry { + filename: string + ts: number // ms epoch + trigger: 'scheduled' | 'pre_promote' | 'manual' + size: number +} + export interface LivingUISettingsProject { id: string name: string @@ -11,55 +25,130 @@ export interface LivingUISettingsProject { port: number | null backendPort: number | null path: string + projectType?: string autoLaunch: boolean logCleanup: boolean + backupsEnabled: boolean + backupInterval: 'hourly' | '6h' | 'daily' | 'weekly' + backupKeep: number + backupStatus?: LivingUIBackupStatus } interface LivingUiSettingsState { // Per-project settings list from `living_ui_settings_get`. projects: LivingUISettingsProject[] hasLoadedProjects: boolean + // Backup dirs of deleted projects (kept on delete by default — D5). + backupOrphans: string[] + // Per-project backup archive list from `living_ui_backups_list`. + backupsByProject: Record + // Per-project in-flight marker for "Back up now" / restore buttons. + backupBusy: Record } const initialState: LivingUiSettingsState = { projects: [], hasLoadedProjects: false, + backupOrphans: [], + backupsByProject: {}, + backupBusy: {}, } const livingUiSettingsSlice = createSlice({ name: 'livingUiSettings', initialState, reducers: { - setSettings(state, action: PayloadAction) { - state.projects = action.payload + setSettings( + state, + action: PayloadAction<{ + projects: LivingUISettingsProject[] + backupOrphans: string[] + }>, + ) { + state.projects = action.payload.projects + state.backupOrphans = action.payload.backupOrphans state.hasLoadedProjects = true }, - // Optimistic per-project setting flip so the toggle doesn't lag on the + // Optimistic per-project setting flip so the control doesn't lag on the // round-trip back from the backend. updateProjectSetting( state, action: PayloadAction<{ projectId: string - setting: 'autoLaunch' | 'logCleanup' - value: boolean + setting: + | 'autoLaunch' + | 'logCleanup' + | 'backupsEnabled' + | 'backupInterval' + | 'backupKeep' + value: boolean | string | number }>, ) { const p = state.projects.find(x => x.id === action.payload.projectId) - if (p) p[action.payload.setting] = action.payload.value + if (p) (p as any)[action.payload.setting] = action.payload.value + }, + setProjectBackups( + state, + action: PayloadAction<{ projectId: string; backups: LivingUIBackupEntry[] }>, + ) { + state.backupsByProject[action.payload.projectId] = action.payload.backups + }, + setBackupBusy( + state, + action: PayloadAction<{ projectId: string; busy: boolean }>, + ) { + state.backupBusy[action.payload.projectId] = action.payload.busy }, }, }) -export const { setSettings, updateProjectSetting } = - livingUiSettingsSlice.actions +export const { + setSettings, + updateProjectSetting, + setProjectBackups, + setBackupBusy, +} = livingUiSettingsSlice.actions export default livingUiSettingsSlice.reducer // --- inbound message handlers -------------------------------------------- register('living_ui_settings_get', (data, dispatch) => { - const d = data as { success: boolean; projects?: LivingUISettingsProject[] } - if (d.success) dispatch(setSettings(d.projects || [])) + const d = data as { + success: boolean + projects?: LivingUISettingsProject[] + backupOrphans?: string[] + } + if (d.success) + dispatch( + setSettings({ + projects: d.projects || [], + backupOrphans: d.backupOrphans || [], + }), + ) +}) + +register('living_ui_backups_list', (data, dispatch) => { + const d = data as { projectId?: string; backups?: LivingUIBackupEntry[] } + if (d.projectId) + dispatch( + setProjectBackups({ projectId: d.projectId, backups: d.backups || [] }), + ) +}) + +// backup_now / restore results clear the busy flag; the archive list and +// settings status line arrive via the follow-up broadcasts the backend +// already sends (living_ui_backups_list; the card refetches settings). +register('living_ui_backup_now_result', (data, dispatch) => { + const d = data as { projectId?: string } + if (d.projectId) + dispatch(setBackupBusy({ projectId: d.projectId, busy: false })) +}) + +register('living_ui_backup_restore_result', (data, dispatch) => { + const d = data as { projectId?: string } + if (d.projectId) + dispatch(setBackupBusy({ projectId: d.projectId, busy: false })) }) // Project setting update response is intentionally not registered here: the diff --git a/app/ui_layer/settings/living_ui_settings.py b/app/ui_layer/settings/living_ui_settings.py index b6fc4f36..128bf967 100644 --- a/app/ui_layer/settings/living_ui_settings.py +++ b/app/ui_layer/settings/living_ui_settings.py @@ -22,6 +22,22 @@ def get_living_ui_projects() -> Dict[str, Any]: projects = [] for project in manager.list_projects(): + # Backup status (spec living-ui-backups-plan Phase 4): sidecar + # last-run state + store totals. Fail-open — a status hiccup + # must not blank the settings page. + backup_status: Dict[str, Any] = {} + try: + from app.factory.host_craftbot import get_factory_host + + state = get_factory_host().backup_state(project.id) + backup_status = { + "lastAt": state["last_at"], + "lastError": state["last_error"], + "count": len(manager.backups.store.list_backups(project.id)), + "totalSize": manager.backups.store.total_size(project.id), + } + except Exception: + backup_status = {} projects.append( { "id": project.id, @@ -30,12 +46,25 @@ def get_living_ui_projects() -> Dict[str, Any]: "port": project.port, "backendPort": project.backend_port, "path": project.path, + "projectType": getattr(project, "project_type", "native"), "autoLaunch": project.auto_launch, "logCleanup": project.log_cleanup, + "backupsEnabled": project.backups_enabled, + "backupInterval": project.backup_interval, + "backupKeep": project.backup_keep, + "backupStatus": backup_status, } ) - return {"success": True, "projects": projects} + # Orphan backup dirs (project deleted, archives kept — D5): listed + # for manual cleanup, never auto-reaped. + orphans = [] + try: + orphans = manager.backups.store.orphan_dirs(manager.projects.keys()) + except Exception: + pass + + return {"success": True, "projects": projects, "backupOrphans": orphans} except Exception as e: return {"success": False, "error": str(e), "projects": []} @@ -66,6 +95,25 @@ def update_project_setting(project_id: str, setting: str, value: Any) -> Dict[st project.auto_launch = bool(value) elif setting == "logCleanup": project.log_cleanup = bool(value) + elif setting == "backupsEnabled": + project.backups_enabled = bool(value) + elif setting == "backupInterval": + if value not in ("hourly", "6h", "daily", "weekly"): + return {"success": False, "error": f"Invalid interval: {value!r}"} + project.backup_interval = value + elif setting == "backupKeep": + try: + keep = int(value) + except (TypeError, ValueError): + return {"success": False, "error": f"Invalid keep count: {value!r}"} + if not 1 <= keep <= 30: + return {"success": False, "error": "Keep count must be 1-30"} + project.backup_keep = keep + # Shrinking retention applies immediately, not at the next backup. + try: + manager.backups.store.prune(project_id, "scheduled", keep) + except Exception: + pass else: return {"success": False, "error": f"Unknown setting: {setting}"} diff --git a/environment.yml b/environment.yml index cd2c3d6e..74c75e02 100644 --- a/environment.yml +++ b/environment.yml @@ -15,6 +15,15 @@ dependencies: - pytesseract=0.3.13 - tesseract=5.5.2 - aiohttp=3.13.3 + # PINNED: openssl 3.6.3 / 3.5.7 regress the Windows cert-store load + # (ssl.SSLError ASN1: NOT_ENOUGH_DATA in _load_windows_store_certs, crashes + # aiohttp at import). Broke 2026-06-22 and AGAIN 2026-08-19 when a nodejs + # install transitively bumped it — keep this pin, verify before raising: + # conda run -n craftbot python -c "import ssl; ssl.create_default_context()" + - openssl=3.6.2 + # Living UI builds: the lui CLI is TypeScript run by Node's native type + # stripping — needs Node >= 24 (older majors ERR_UNKNOWN_FILE_EXTENSION). + - nodejs>=24 - beautifulsoup4=4.14.3 - chardet=5.2.0 - lxml=6.0.2 diff --git a/living-ui/blueprint/pb/pb_hooks/_a2app.pb.js b/living-ui/blueprint/pb/pb_hooks/_a2app.pb.js index d94683c0..8e1f07a5 100644 --- a/living-ui/blueprint/pb/pb_hooks/_a2app.pb.js +++ b/living-ui/blueprint/pb/pb_hooks/_a2app.pb.js @@ -68,6 +68,10 @@ routerAdd('GET', '/api/_a2app', (e) => { livingUIVersion: manifest.livingUIVersion || null, kitVersion: manifest.kitVersion || null, }, + // Which environment this instance IS: the dev provisioner stamps + // env:"dev" into its copy's manifest; anything else is the live app. + // Structural, so a client never has to guess which DB a port holds. + env: manifest.env === 'dev' ? 'dev' : 'live', schemaVersion: a2.schemaVersion(e.app), serverNow: a2.serverNowIso(), serverTzOffsetMinutes: -new Date().getTimezoneOffset(), diff --git a/living-ui/blueprint/pb/pb_hooks/_a2app_lib.js b/living-ui/blueprint/pb/pb_hooks/_a2app_lib.js index 99db72be..e360e668 100644 --- a/living-ui/blueprint/pb/pb_hooks/_a2app_lib.js +++ b/living-ui/blueprint/pb/pb_hooks/_a2app_lib.js @@ -29,7 +29,7 @@ * missing required -> 400 validation_required */ -var ADAPTER_VERSION = '1.7.1'; +var ADAPTER_VERSION = '1.8.0'; var RECORD_PATH = /^\/api\/collections\/([^\/]+)\/records(\/([^\/?]+))?$/; function rules() { diff --git a/mkdocs/docs/living-ui/a2app-protocol.md b/mkdocs/docs/living-ui/a2app-protocol.md index 65b689c1..5fcb8320 100644 --- a/mkdocs/docs/living-ui/a2app-protocol.md +++ b/mkdocs/docs/living-ui/a2app-protocol.md @@ -43,6 +43,7 @@ Unauthenticated, and more than a greeting. PocketBase answers **HTTP 200 for any | `app.id` | Writing to the wrong app. Identity survives a port change; confirm it matches the app the user meant | | `protocol` / `adapterVersion` | Contract versus implementation. The contract stays stable while the adapter gains fixes; a client can detect a known bug or a stale app | | `pbVersion` | The filter grammar is PocketBase's and therefore part of this contract; this says which dialect you get | +| `env` | Writing to the wrong environment. `"dev"` = a disposable dev instance (fresh schema-only DB, destroyed at promote); `"live"` = the real app and its real data. A client that means to create test records must see `"dev"`; one storing user data must see `"live"` | | `schemaVersion` | Writing against a stale schema. Cache `describe` against this fingerprint and re-fetch when it changes | | `serverNow` / `serverTzOffsetMinutes` | The app's clock and zone, so a client can tell whether its own clock agrees before sending date-based writes | diff --git a/mkdocs/docs/living-ui/framework.md b/mkdocs/docs/living-ui/framework.md index 8dbe89c3..37c968da 100644 --- a/mkdocs/docs/living-ui/framework.md +++ b/mkdocs/docs/living-ui/framework.md @@ -109,7 +109,7 @@ flowchart LR ``` - **The validation gate** runs before anything boots: TypeScript must compile, the frontend must build, migrations must apply on a fresh database, the operations manifest must validate and route correctly, and the system-managed files must be untouched. Errors come back source-annotated, and a circuit breaker stops a build that keeps failing on the identical error. -- **walk_verify** is a [sub-agent](../core/concepts/sub-agents.md) that opens the running app in a headless browser and exercises it feature by feature against `reference/requirements.md`, folding server-side errors from `pocketbase.log` into its defect reports. Its verdict is `pass`, `incomplete`, `defects`, `blocked`, or `unparseable`, and a clean pass is the **only** way a build completes. On a first build, a pass marks the app delivered; on an evolution, it flips the staging copy live (see [Managing apps](managing.md#evolving-an-app)). +- **walk_verify** is a [sub-agent](../core/concepts/sub-agents.md) that opens the running app in a headless browser and exercises it feature by feature against `reference/requirements.md`, folding server-side errors from `pocketbase.log` into its defect reports. Its verdict is `pass`, `incomplete`, `defects`, `blocked`, or `unparseable`, and a clean pass is the **only** way a change completes. Every change is verified in the dev environment (the new code on a hidden port with a fresh, schema-only database) and a pass **promotes** it: on a first build the live database is created fresh from the migration chain; on an evolution the new migrations apply to the real data at boot (see [Managing apps](managing.md#evolving-an-app)). The principle behind both gates, and behind the [protocol](a2app-protocol.md) itself: **a property that matters is enforced by the system, not requested of the model.** An app that does not demonstrably work in a real browser is not announced as working. diff --git a/mkdocs/docs/living-ui/index.md b/mkdocs/docs/living-ui/index.md index dd692027..43d4505c 100644 --- a/mkdocs/docs/living-ui/index.md +++ b/mkdocs/docs/living-ui/index.md @@ -22,7 +22,7 @@ Two pieces make that trustworthy: --- - Operating a delivered app's data and verbs, evolving it safely through a staging copy, restarting, importing, converting foreign apps, and the marketplace. + Operating a delivered app's data and verbs, evolving it safely through a dev environment, restarting, importing, converting foreign apps, and the marketplace.
@@ -33,7 +33,7 @@ The agent's relationship with a Living UI has three distinct capabilities, and t | Capability | What it means | What guarantees it | |---|---|---| | **Build** | Turn a requirements interview into a working app: schema, verbs, UI | The validation gate plus browser verification; an app that does not demonstrably work is never announced as working | -| **Evolve** | Change a delivered app's code and schema on request | A staging copy with cloned data; the live app is replaced only by a verified successor | +| **Evolve** | Change a delivered app's code and schema on request | A dev environment with a fresh schema-only DB (live data is never cloned); the live app is replaced only by a verified successor | | **Operate** | Act on the app's data and declared verbs in seconds ("add a todo for tomorrow" becomes a row) | The A2App protocol: schema discovery, write guards, and system-authored receipts | The distinction between operating and evolving is decided per request by the agent, and it matters: a data write never triggers a rebuild, and a code change never touches live data until it verifies. See [Managing apps](managing.md). @@ -54,7 +54,7 @@ flowchart LR G --> W["walk_verify
real browser, every feature"] W --> D(["Delivered
live URL, sidebar tab"]) D --> OP["Operate
data + declared verbs"] - D --> MOD["Evolve
staging copy → verify → live"] + D --> MOD["Evolve
dev env → verify → promote"] MOD --> G ``` diff --git a/mkdocs/docs/living-ui/managing.md b/mkdocs/docs/living-ui/managing.md index cc6c3cc2..7d7e3672 100644 --- a/mkdocs/docs/living-ui/managing.md +++ b/mkdocs/docs/living-ui/managing.md @@ -1,6 +1,6 @@ # Managing apps -A delivered Living UI is a live application with your real data in it. Everything that happens to it afterward falls into two categories with very different mechanics: **operating** (data and verb calls through the [A2App protocol](a2app-protocol.md): instant, no rebuild) and **evolving** (code and schema changes, which go through a staging copy and full re-verification before they touch the live app). This page covers both, plus restarting, importing, converting foreign apps, the marketplace, and multi-agent use. +A delivered Living UI is a live application with your real data in it. Everything that happens to it afterward falls into two categories with very different mechanics: **operating** (data and verb calls through the [A2App protocol](a2app-protocol.md): instant, no rebuild) and **evolving** (code and schema changes, which go through a dev environment and full re-verification before they touch the live app). This page covers both, plus restarting, importing, converting foreign apps, the marketplace, and multi-agent use. ## Operate or evolve @@ -11,7 +11,7 @@ The agent decides which category a request is, per request; nothing is routed in | "add a todo for tomorrow" | Operate | One validated write. Seconds | | "clear all the done items" | Operate | One declared operation, confirmed first if marked destructive | | "summarise this week's entries" | Operate | Reads plus (if the app declares one) an operation | -| "add a priority filter to the board" | Evolve | Staging copy, code, validation gate, browser verification, then live | +| "add a priority filter to the board" | Evolve | Dev environment, code, validation gate, browser verification, then promote | The boundary is enforced, not just encouraged. Getting it wrong used to be expensive: a data write that triggers the build machinery rebuilds a live app and drives a browser over your real records. Build skills therefore load **per run**, chosen by the agent from the request, and a plain write never touches them. @@ -45,19 +45,31 @@ Code changes to a delivered app never touch it directly: ```mermaid flowchart LR - REQ["Change request"] --> STG["Staging copy
cloned data, hidden port"] + REQ["Change request"] --> STG["Dev environment
fresh schema-only DB, hidden port"] STG --> CODE["Agent edits code
+ appends to requirements.md"] CODE --> GATE["Validation gate"] --> WV["walk_verify
headless browser"] - WV -->|pass| FLIP["Staging flips live"] + WV -->|pass| FLIP["Promote: live boots the new code"] WV -.->|defects| CODE ``` -- The agent loads a build skill for the run, works on a **staging copy** with a disposable clone of the app's data on a hidden port, and follows the same [build loop](framework.md#how-the-agent-builds) as a first build: schema migrations first, operation declarations, kit-composed UI, gate after every meaningful change. +- The agent loads a build skill for the run, works in a **dev environment** — a disposable copy of the app's code on a hidden port whose database is rebuilt fresh from the migration chain (your real data is never cloned into it) — and follows the same [build loop](framework.md#how-the-agent-builds) as a first build: schema migrations first, operation declarations, kit-composed UI, gate after every meaningful change. - The change is appended to `reference/requirements.md` under `## Changes`, keeping the binding spec current; verification checks the app against that file, so a stale spec would produce a wrong verdict. -- Only a clean verification verdict flips staging to live. A failed change never replaces the working app, and your real data is never the test bed. +- Only a clean verification verdict promotes the change to live. A failed change never replaces the working app, and your real data is never the test bed — it never even enters the environment being tested. Mid-arc writes to the live app's real data are refused while an evolution is in flight, so the two paths cannot interleave. +## Backups + +Every native app's live data (database + uploaded files) is backed up automatically — **daily, keeping the last 7**, by default. Configure it per app in **Settings → Living UI**: switch scheduled backups off, pick a frequency (hourly / 6 h / daily / weekly), set how many to keep, or take a manual backup with **Back up now**. Three kinds of archives accumulate: + +- **Scheduled** — taken on the interval you chose; the oldest beyond your keep-count are pruned automatically. +- **Pre-update** — taken automatically right before every code change is deployed to an app with live data (the last 3 are kept). If this backup fails, the deploy is aborted rather than risked. +- **Manual** — taken with the button; never removed automatically. + +Archives live outside the app's own directory (`living_ui/_backups/`), so they survive anything that happens to the app — including deleting it: a deleted app's backups are kept and listed under **Leftover backups** in the same settings tab until you remove them yourself. Backups never leave your machine and are not part of project exports. + +**Restoring** (from the app's backup list in settings) returns the app to the archived state: data created after that point is removed, but the current state is backed up first — so a restore can itself be undone. Restore is a user action only; the agent cannot trigger it. + ## Restarting Ask the agent to restart an app (or use its tab). A restart runs the full launch pipeline: dependency check, validation gate, boot (PocketBase plus frontend), health check. Launch also re-stamps the [A2App adapter](a2app-protocol.md) and refreshes the agent token, which is how apps a user already had pick up adapter fixes; delivery at create, install, import, **and every launch** is what keeps the whole installed base current. diff --git a/skills/living-ui-creator/SKILL.md b/skills/living-ui-creator/SKILL.md index 36e4df59..ab0146f8 100644 --- a/skills/living-ui-creator/SKILL.md +++ b/skills/living-ui-creator/SKILL.md @@ -260,10 +260,13 @@ only where agent judgment adds value — plain code handles plain events. ## Finish: launch, then verify 1. `living_ui_notify_ready(project_id="")` — runs the gate - (**types → build → migrations-on-fresh-db → ops → ownership**), starts the - app and health-checks it. On errors: read ALL of them, fix ALL of them, - call it again. Success = app RUNNING but NOT yet verified. Never start - servers manually. + (**types → build → migrations-on-fresh-db → ops → ownership**), then + starts your code in the DEV environment (a copy on a hidden port with a + fresh post-migration DB) and health-checks it. Its message gives you the + dev URL and dev dir — test and read logs THERE; keep editing in the real + project dir (each notify_ready syncs your edits in). On errors: read ALL + of them, fix ALL of them, call it again. Success = app RUNNING (in dev) + but NOT yet verified. Never start servers manually. 2. **REALITY CHECK — look at what actually exists, not at what you wrote.** Success messages lie by omission; stored state does not. While the app runs: @@ -289,14 +292,15 @@ only where agent judgment adds value — plain code handles plain events. completes the build.** Failing features come back as a report: fix them, then repeat step 1 and step 3. -Test data is fine during the build: at delivery the platform resets the -app's data to its pristine post-migration state, so records you or the -verifier created never reach the user. Data your migrations SEED survives -(they re-run on the clean DB) — put anything the user must see on first -open in a migration, never insert it by hand. Externally-fetched data is -reset too: an app that syncs from an API must self-populate on an empty -DB (fetch at boot or when the collection is empty — never rely on a sync -that happened during the build). +Test data is fine during the build: you are working in the DEV environment, +whose database is disposable — at delivery the platform boots the LIVE app +with a fresh database built purely from your migrations, so records you or +the verifier created never reach the user. Data your migrations SEED +survives (they run on the fresh live DB) — put anything the user must see +on first open in a migration, never insert it by hand. Externally-fetched +data does not carry over either: an app that syncs from an API must +self-populate on an empty DB (fetch at boot or when the collection is +empty — never rely on a sync that happened during the build). **HONESTY RULE:** the app is ready ONLY when `living_ui_walk_verify` returns `status: success`. If you cannot make it pass, tell the user the build @@ -308,11 +312,13 @@ the app fetched it from the real source. - Full platform reference (bridge, jobs, kit API): `living-ui/docs/agent-guide.md` (repo-level, read on demand). -- Frontend runtime errors: `{project_path}/logs/frontend_console.log` - (console.error/warn + uncaught errors are auto-relayed). -- Server: `{project_path}/logs/pocketbase.log`. -- Data inspection: the PB REST API on the project's port - (`GET /api/collections//records`). +- The RUNNING instance is the dev copy — its logs live in the dev dir that + `living_ui_notify_ready` reported, not in the project dir: + `{dev_dir}/logs/frontend_console.log` (console.error/warn + uncaught + errors are auto-relayed) and `{dev_dir}/logs/pocketbase.log`. +- Data inspection: the PB REST API on the dev port notify_ready returned + (`GET /api/collections//records`). `GET /api/_a2app` answers + `env: "dev"` if you need to confirm which instance a port is. ## FORBIDDEN diff --git a/skills/living-ui-modify/SKILL.md b/skills/living-ui-modify/SKILL.md index e154bb74..793e6f8d 100644 --- a/skills/living-ui-modify/SKILL.md +++ b/skills/living-ui-modify/SKILL.md @@ -65,24 +65,31 @@ this skill covers only what differs. ## Finish ``` -living_ui_notify_ready(project_id="") # gate + boot STAGING copy -living_ui_walk_verify(project_id="") # verify staging + DEPLOY +living_ui_notify_ready(project_id="") # gate + boot DEV env +living_ui_walk_verify(project_id="") # verify dev + PROMOTE ``` -On a delivered app these run in **staging mode**: `notify_ready` gates and -boots a disposable COPY of the app (code + cloned data) on a hidden port — -the user's live app keeps running the previous version, untouched. Test -freely against the staging URL it returns: every record you create there is -thrown away. `walk_verify` drives the staging copy in a real (headless) -browser; a clean verdict is what DEPLOYS your change to the live app (new -migrations apply to the real data at boot) and announces it. +These run in the **dev environment**: `notify_ready` gates and boots a +disposable copy of your new CODE on a hidden port with a **FRESH, EMPTY +database** — migrations replay at boot, so only data your migrations seed +exists. The user's live app keeps running the previous version, untouched, +and its data is NEVER cloned into dev. Test freely against the dev URL it +returns (create whatever test records you need — they are thrown away). +`walk_verify` drives the dev instance in a real (headless) browser; a clean +verdict is what PROMOTES your change to the live app (new migrations apply +to the real data at its boot) and announces it. -- **Never run `lui validate` or `lui dev` against the real project dir of a - delivered app** — the build overwrites the served frontend in place and - blanks the user's live UI. `notify_ready` gates the staging copy for you. +- **The dev DB starts empty every time.** If a feature needs data to be + visible, either seed it in a migration (survives promote) or create test + records through the app/API after `notify_ready` (dev-only, disposable). +- **Never run `lui validate` or `lui dev` against the real project dir** — + the build overwrites the served frontend in place and blanks the user's + live UI. `notify_ready` gates the dev copy for you. - **Never write test data to the live app** (its DB is the user's real - data; writes outside staging are refused). Do all testing after - `notify_ready`, against the staging URL. + data; agent test writes outside the dev env are refused). Do all testing + after `notify_ready`, against the dev URL. `GET /api/_a2app` answers + `env: "dev"` or `env: "live"` if you need to confirm which instance a + port is. HONESTY RULE: the change is live only when `living_ui_walk_verify` returns `status: success` — never tell the user a change is live when the relaunch,