From f0be5eb725ed832a32efbdba428f049ce01bbe08 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 05:23:07 -0600 Subject: [PATCH 01/12] feat(supervise): preserve durable execution identity --- docs/api/primitive-catalog.md | 70 +- src/durable/spawn-journal.ts | 190 +- src/mcp/index.ts | 7 + src/mcp/tools/coordination.ts | 657 ++++-- src/runtime/index.ts | 48 +- src/runtime/supervise/budget.ts | 238 ++- src/runtime/supervise/coordination-driver.ts | 134 +- src/runtime/supervise/coordination-mcp.ts | 25 +- src/runtime/supervise/deadline.ts | 110 + src/runtime/supervise/driver-executor.ts | 253 ++- src/runtime/supervise/event-bus.ts | 31 +- src/runtime/supervise/run-context.ts | 14 +- src/runtime/supervise/runtime.ts | 1763 ++++++++++++----- src/runtime/supervise/scope.ts | 1184 +++++++++-- src/runtime/supervise/snapshot.ts | 18 + src/runtime/supervise/supervise.ts | 1120 ++++++++++- src/runtime/supervise/supervisor-agent.ts | 325 ++- src/runtime/supervise/supervisor.ts | 487 ++++- src/runtime/supervise/types.ts | 271 ++- tests/helpers/supervisor-resume-child.ts | 6 +- tests/helpers/supervisor-wait-child.ts | 5 + tests/kernel/coordination-mcp.test.ts | 63 + tests/kernel/coordination.test.ts | 306 ++- tests/kernel/event-bus.test.ts | 24 + tests/kernel/supervise-convenience.test.ts | 543 ++++- tests/kernel/supervise-deadline.test.ts | 254 +++ .../supervise-full-profile-bridge.test.ts | 808 ++++++++ .../supervise-restart-resource-safety.test.ts | 1295 ++++++++++++ tests/kernel/supervisor-agent.test.ts | 244 ++- .../runtime/executor-config-snapshot.test.ts | 266 +++ 30 files changed, 9546 insertions(+), 1213 deletions(-) create mode 100644 src/runtime/supervise/deadline.ts create mode 100644 src/runtime/supervise/snapshot.ts create mode 100644 tests/kernel/supervise-deadline.test.ts create mode 100644 tests/kernel/supervise-full-profile-bridge.test.ts create mode 100644 tests/kernel/supervise-restart-resource-safety.test.ts create mode 100644 tests/runtime/executor-config-snapshot.test.ts diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 5c52773a..85991181 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -176,7 +176,7 @@ Import from `@tangle-network/agent-runtime` — 401 exports. | `AgentCandidateRepositoryPort` | interface | Resolves a declared GitHub repository to an already-present local Git object store. | | `AgentCandidateTaskExecution` | interface | Runtime placement for one exact cell from a signed candidate experiment. | | `AgentCandidateWorkspacePort` | interface | Materializes an already-verified workspace archive. | -| `AgentSpec` | interface | `AgentProfile` does NOT carry a `harness`/backend field — `harness` lives on the | +| `AgentSpec` | interface | `AgentProfile.harness` is a portable preference; this wrapper records the executor decision for | | `BackendErrorDetail` | interface | Typed transport / backend failure detail. Carried on `backend_error` and | | `BackendRetryPolicy` | interface | Retry policy for transient transport errors (rate limits, upstream | | `Budget` | interface | A budget envelope on a spawn or the root. All ceilings; the pool reserves against them. | @@ -254,7 +254,7 @@ Import from `@tangle-network/agent-runtime` — 401 exports. ### Vertical agent — manifest + surface proposal source -Import from `@tangle-network/agent-runtime/agent` — 41 exports. +Import from `@tangle-network/agent-runtime/agent` — 48 exports. | Symbol | Kind | Summary | |---|---|---| @@ -264,6 +264,7 @@ Import from `@tangle-network/agent-runtime/agent` — 41 exports. | `createSurfaceImprovementProposer` | function | Resolve each finding to a real surface and draft a detached patch candidate. | | `defineAgent` | function | Construct a validated agent manifest. Throws `AgentManifestError` | | `defineProfileMaterializationContract` | function | Define the profile axes a concrete run path actually carries into execution. | +| `profileMaterializationAxes` | function | Return the exact canonical axes a complete profile actually requests. Compound prompt, model, | | `renderProfileMaterializationIssues` | function | Format profile-axis drop issues into a concise operator-facing error. | | `renderSurfaceIssues` | function | Format a list of surface validation issues into a human-readable error string. | | `resolveSubjectPath` | function | Resolve a parsed `FindingSubject` to the file path the substrate | @@ -271,9 +272,14 @@ Import from `@tangle-network/agent-runtime/agent` — 41 exports. | `validateProfileMaterialization` | function | Return every changed profile axis that the selected run path would drop. | | `validateSurfaces` | function | Validate an `AgentSurfaces` map on disk — missing paths fail loud at `defineAgent` time instead of silently skipping self-improvement edits. | | `AGENT_PROFILE_MATERIALIZATION_AXES` | const | Known AgentProfile axes a run path may or may not carry into execution. | +| `controlProfileMaterialization` | const | Materialization contract for a raw process path that carries only control/identity fields. | +| `fullProfileMaterialization` | const | Materialization contract for a run path that executes every canonical AgentProfile axis. | +| `promptControlProfileMaterialization` | const | Materialization contract for an injected inference function whose surrounding driver still | +| `promptModelProfileMaterialization` | const | Materialization contract for an intentionally limited prompt-and-model execution path. | | `promptOnlyProfileMaterialization` | const | Materialization contract for a run path that only injects prompt text. | | `promptResourceProfileMaterialization` | const | Materialization contract for a run path that injects prompt text plus inline resources. | | `sandboxActProfileMaterialization` | const | Materialization contract for `createSandboxAct`, which forwards the full AgentProfile. | +| `worktreeCliProfileMaterialization` | const | Materialization contract for a local coding CLI in an isolated git worktree. | | `AgentManifestError` | class | Thrown when `defineAgent` finds a required surface missing on disk. | | `AgentManifest` | interface | The full agent manifest. Each agent ships ONE of these. | | `AgentSurfaces` | interface | Surface declarations. Every path is repo-relative (or absolute) at | @@ -285,6 +291,7 @@ Import from `@tangle-network/agent-runtime/agent` — 41 exports. | `SurfaceValidationIssue` | interface | Validate that every declared surface exists on disk under `repoRoot`. | | `ValidateProfileMaterializationOptions` | interface | Input for checking a candidate diff against a run path. | | `AgentProfileMaterializationAxis` | type | AgentProfile axis name, with `custom:` reserved for caller-owned extensions. | +| `CanonicalAgentProfileMaterializationAxis` | type | Canonical AgentProfile axes used when checking one complete profile. | **Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentRubric`, `AgentRunContext`, `AgentRunInvocation`, `AgentRuntime`, `AnalystConfig`, `CreateSandboxActOptions`, `CreateSurfaceImprovementProposerOptions`, `DraftPatchInput`, `DraftPatchOutput`, `JudgeConfig`, `ResolvedSurface`, `RubricDimension`, `SurfaceImprovementEdit`, `KnownAgentProfileMaterializationAxis`. @@ -492,7 +499,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 594 exports. +Import from `@tangle-network/agent-runtime/kernel` — 624 exports. | Symbol | Kind | Summary | |---|---|---| @@ -505,10 +512,11 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `areaUnderCurve` | function | Mean of a best-so-far curve — the anytime AUC when the curve is normalized to [0,1]. Higher = | | `asAuthoredProfile` | function | Narrow an untyped `spawn_agent` profile argument to an `AuthoredProfile`, or null if the | | `assertModelAllowed` | function | Throw a `ConfigError` when `allowed` is set, `model` is defined, and `model` is not a | +| `assertProfileModelsAllowed` | function | Check every canonical model-bearing field in a complete profile, including the models a | | `assertStrategyContract` | function | Static CONTRACT lint over an authored strategy module — the module-boundary | | `assessAuthoredProfile` | function | OBSERVE one authored `AgentProfile` and score its richness (no judge verdict is read). The task | | `auditIntent` | function | The route-rigor analyst: compare declared vs revealed vs user intent over a trajectory and return aligned / drifting / diverged with evidence and one recommended intervention. | -| `authoredWorker` | function | Build a worker AGENT from a profile the supervisor authored: the authored `systemPrompt` + | +| `authoredWorker` | function | Build a router-only worker from an authored profile. This helper executes the prompt/model axes; | | `authorStrategy` | function | Author + load a strategy from losses. Throws when the author emits no loadable module; | | `bestSoFar` | function | The best-so-far fold — the ONE definition of "how good was the run after k results", shared by | | `breadthStrategy` | function | BREADTH: K independent rollouts (each own artifact), verifier picks the best. | @@ -674,6 +682,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `builtinShapes` | const | The default registry `runPersonified` resolves a shape name against. Empty by construction — | | `cliWorktreeExecutor` | const | The leaf `createWorktreeCliExecutor` as a backend-as-data factory: a supervisor-authored | | `collectDelivered` | const | Every verified distinct output, highest score first — the shape for competing hypotheses, a | +| `DEFAULT_AUTHORED_PROFILE_SECURITY_POLICY` | const | Manager-authored profiles are untrusted until product policy says otherwise. Remote MCP and | | `DEFAULT_AWAIT_EVENT_TIMEOUT_MS` | const | Default ceiling for a single `await_event` block (ms). Chosen well under any reasonable remote | | `DEFAULT_SANDBOX_STEERING_MAX_TURNS` | const | Ceiling on continuation turns. Turn 0 is the task; every later turn is a folded steer, so | | `DEFAULT_STALL_AFTER_MS` | const | How long a worker may produce no metered activity before a `progress()` read calls it stalled. | @@ -705,18 +714,21 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `ActivityNote` | interface | The most recent activity the executor can name — one tool call, one turn, or a free-form note. | | `Agent` | interface | One self-similar atom. A leaf is an `Agent` that never calls `scope.spawn`; a driver | | `AgentEnvironmentProviderRegistry` | interface | In-memory registry for named `AgentEnvironmentProvider` instances. | +| `AgentExecutionRef` | interface | Caller-owned identity beyond the exact profile/task bytes Scope can compute itself. | | `AgenticSurface` | interface | A stateful, checkable environment an agent operates over with tools. Open behind one interface. | | `AgentProfile` | interface | Public provider-neutral agent profile contract. | | `AgentRunSpec` | interface | Sandbox-SDK-shaped agent specification. | -| `AgentSpec` | interface | `AgentProfile` does NOT carry a `harness`/backend field — `harness` lives on the | +| `AgentSpec` | interface | `AgentProfile.harness` is a portable preference; this wrapper records the executor decision for | | `AgentTurnUsage` | interface | Metered usage of one turn, summed over every cost-bearing event the backend | | `AnalystFinding` | interface | Unified envelope every analyst emits. Schema-versioned so renderers | | `AnalystFindingEvent` | interface | A trace-analyst result re-entered as a message on the bus (the `finding` event kind). | -| `AuthoredProfile` | interface | What the supervisor AUTHORS per sub-task — a worker recipe (a partial `AgentProfile`). | +| `AuthorizedDownMessage` | interface | Product-authorized continuation bytes. Returning a narrowed instruction replaces the proposed | +| `AuthorizedSpawn` | interface | The product-authorized result for one complete spawn request. Attribution is never accepted | | `BenchmarkCell` | interface | One strategy's outcome on one task — the per-task cell an optimizer consumes. | | `BenchmarkReport` | interface | Benchmark output: per-strategy means plus the full per-task × per-strategy losses table an optimizer mines. | | `BridgeSeam` | interface | cli-bridge seam. A local OpenAI-compatible bridge that fronts harness CLIs | | `Budget` | interface | A budget envelope on a spawn or the root. All ceilings; the pool reserves against them. | +| `BudgetPoolRestore` | interface | State recovered from a prior process before new work is admitted. `committed` is measured spend | | `BusEvent` | interface | Every bus event is a discriminated union member keyed by `type`. | | `BusRecord` | interface | A published event stamped for ordering and observability. `seq` is the monotonic publish index; | | `CheckExecChannel` | interface | Minimal exec channel the default runner needs. `SandboxInstance` (and therefore | @@ -733,6 +745,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `CompletionPolicy` | interface | When a verdict authorizes the driver to END. Deterministic → trust (ground truth); | | `CompletionVerdict` | interface | The "is it done?" verdict an analyst returns to the parent. | | `ConcurrencyCaps` | interface | The caps a host can set on simultaneous work. See the ledger in this module's header for what | +| `ContinuationInstruction` | interface | Durable authorization receipt written before a continuation reaches a worker. | | `CoordinationLog` | interface | The durable coordination side-log seam. `append` records one bus event (kinds it does not | | `Corpus` | interface | The durable cross-run corpus — the learning-flywheel store. DISTINCT from `SpawnJournal` | | `CorpusFilter` | interface | A corpus query filter — every field is an AND-narrowing; an omitted field does not constrain. | @@ -746,14 +759,20 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `DeliverableSpec` | interface | The deployable completion oracle passed to {@link gateOnDeliverable}: a `check` that | | `DeliveredOutput` | interface | One DELIVERED child, materialized: settled `done`, oracle-passed, output rehydrated. `out` is | | `DispatchUnit` | interface | One unit of queued work: the agent to run, its task, and the spawn options (budget + label). | -| `DownMessageEvent` | interface | A parent→child message (the down-leg): recorded for observability, delivered via the child inbox, | +| `DownMessageAuthorizationInput` | interface | Detached continuation bytes and exact worker identity presented to product authorization before | +| `DownMessageDeliveryAttempt` | interface | A durable marker written after authorization and immediately before Runtime calls `Scope.send`. | +| `DownMessageEvent` | interface | A parent→child delivery result (the down-leg): recorded for observability, never pulled back by | | `DumbDriverOptions` | interface | Options for {@link dumbDriver}. | | `EqualKArm` | interface | One arm of an equal-k comparison — a labeled trajectory (a `TrajectoryReport` is one arm's whole | | `EqualKOnCostOptions` | interface | `equalKOnCost(arms, { tolerance? })` — assert arms are comparable at EQUAL conserved COST | | `EqualKVerdict` | interface | The equal-k-on-cost verdict: whether every arm spent within `tolerance` of the others on the | | `ExecCtx` | interface | Execution context for `runAgentRounds`: the sandbox client the kernel creates boxes through, plus optional runtime hooks. | | `Executor` | interface | The leaf runtime — ONE open interface, not a closed union. `execute` returns a | +| `ExecutorAccounting` | interface | Split used by a recursive executor when journaled child work differs from the full amount | | `ExecutorContext` | interface | Construction context handed to a `ExecutorFactory` — the seams a built-in needs | +| `ExecutorExecutionBinding` | interface | Volatile execution routing that is true for one attempt but is not profile identity. The full | +| `ExecutorMaterialization` | interface | Data-only declaration from trusted executor code about the exact sealed plan `execute` uses. | +| `ExecutorNodeContext` | interface | Kernel-owned context for the concrete supervised node a factory is constructing. | | `ExecutorProgress` | interface | What an executor OPTIONALLY adds to the scope-derived progress (`Executor.progress()`). Every | | `ExecutorRegistry` | interface | The OPEN resolver: maps an `AgentSpec` to a `ExecutorFactory`. The default | | `ExecutorResult` | interface | Terminal artifact of a one-shot `Executor.execute`. | @@ -788,9 +807,11 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `LoopTokenUsage` | interface | LLM token usage. Structurally maps into agent-eval's paid-call receipt so a | | `LoopUntilSpec` | interface | `loopUntil({ until, step })` — iterative deepening inside the conserved pool: spawn one `step` | | `LoopUntilState` | interface | The accumulated state `loopUntil` threads across rounds — the running candidate + the round | +| `MaterializedExecutionIdentity` | interface | External execution identity that operators can use to join this node to its backend. | | `McpEndpoint` | interface | Where a handle's MCP server lives; headers carry per-artifact scoping. | | `MountManifestEntry` | interface | One mounted resource recorded during box preparation — a pure provenance | | `NaiveDriverOptions` | interface | Options for {@link naiveDriver}. | +| `NodeExecutionIdentity` | interface | Durable identity of one realized node. Missing digests mean the input was not canonical JSON. | | `OpenSandboxRunBeforeStartContext` | interface | Context available after the box/session exists and before the first prompt is | | `OutputAdapter` | interface | Stream of `SandboxEvent`s → typed `Output`. | | `PairwiseVerdict` | interface | One profile pair compared on the scenarios they BOTH ran — the "who actually beat whom" verdict. | @@ -803,7 +824,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `PersonaExecutors` | interface | How a persona supplies executor resolution. Either a pre-built registry (factories already | | `PipelineStage` | interface | `pipeline(stages)` — sequential composition: each stage's `Outcome.deliverable` feeds the next | | `PiSeam` | interface | How to launch pi in its out-of-process RPC mode, and how long to wait on it. | -| `PriorCoordination` | interface | What a prior process's coordination log replays into a resumed driver. | +| `PriorCoordination` | interface | Coordination evidence loaded from prior processes of one durable supervised run. | | `ProfileRichness` | interface | Per-field verdict on one authored profile — the raw material the bench renders + scores. | | `ProfileRichnessThresholds` | interface | Thresholds below which a system prompt is treated as a thin stub. Tunable per call. | | `ProgressSample` | interface | One settled unit of work, reduced to what a stop rule reads. `objective` is the run's own | @@ -858,7 +879,8 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `StructuralRolloutResult` | interface | The body's deliverable — a `StrategyResult` plus selection provenance. The extra | | `SuperviseSurfaceResult` | interface | The deployable outcome of a supervised surface run. | | `Supervisor` | interface | Owns the conserved pool, the spawn log, the abort cascade, the OTP intensity breaker, | -| `SupervisorProfile` | interface | The supervisor's profile — the subset of an `AgentProfile` that selects + shapes its brain. | +| `SupervisorNodeContext` | interface | Trusted run/node identity Runtime binds to one manager. Model-authored tool arguments cannot | +| `SupervisorToolDescriptor` | interface | One product-owned tool. It reuses the canonical MCP descriptor fields while Runtime supplies | | `SurfaceWorkerConfig` | interface | How a worker runs the surface task (its router substrate + per-attempt bounds). | | `SurfaceWorkerOut` | interface | What a surface worker settles with — the surface verdict the driver + deliverable read. `resolved` is | | `ToolLoopCompaction` | interface | Self-compaction — bound the loop's OWN context window the way a fresh-respawn (dumb-Ralph) loop | @@ -876,6 +898,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `WidenLineage` | interface | A lineage the gate may widen toward — the settled child that looked promising + the findings | | `WidenSpec` | interface | `widen({ gate })` (G5) — the STREAMING spawn-on-completion driver. Unlike the static-fanout | | `WorkerProgress` | interface | The full live view of one worker, as `observe_agent` returns it mid-flight. | +| `WorkerSpawnContext` | interface | Immutable task, allocation, identity attribution, and semantic key supplied while a manager's | | `WorktreeCommandResult` | interface | Outcome of one verification command run in the worktree (test or typecheck). | | `WorktreeHarnessResult` | interface | The canonical result of one worktree-harness run, projected by each port to its own shape. | | `WorktreeProfileMaterializationReceipt` | interface | Proof of the profile inputs delivered before the worker process started. | @@ -884,16 +907,22 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `AgentTurnBackend` | type | The execution substrate one turn runs on — a closed discriminated union over | | `ApplyContinuation` | type | Fold a steering string into the caller's Task shape, producing the Task for | | `AssertTraceDerivedFindings` | type | The firewall assertion contract, re-stated for the reactive seam (PORT of | +| `AuthoredProfile` | type | What the supervisor AUTHORS per sub-task: one complete canonical profile whose name and | +| `AuthorizeDownMessage` | type | Product decision over an exact continuation before it is durably recorded or delivered. | | `AxisScoresOf` | type | Decompose ONE record into per-axis scores (e.g. judge dimensions). When set, it REPLACES the | | `BudgetReadout` | type | Post-reservation pool readout — the shape `Scope.budget` exposes. `tokensLeft`, | | `CombinatorShape` | type | A combinator is just a `LoopShape`: a factory `(ShapeContext) => Agent` whose `Agent.act` | +| `CoordinationDeliveryEvidence` | type | Durable delivery evidence retained in commit order. An attempt without a later event carrying | | `CoordinationEvent` | type | Every message on the one typed pipe. UP (child→parent): question / settled / finding — queued for | +| `CoordinationOwnerId` | type | Stable identity of the supervisor that owns one coordination stream. High-level supervision | | `DefinePersona` | type | Builds a frozen `Persona`, failing loud on the executors-supplied invariant (neither a | | `Deliverable` | type | How a typed deliverable `Out` is materialized from a finished turn. | | `DispatchStopReason` | type | Why the dispatcher stopped admitting work. `drained` = the queue ran dry (the ordinary end); | -| `DriveHarness` | type | How to run a sandboxed harness as the DRIVER, with the coordination verbs mounted — the substrate | +| `DownMessageDeliveryOutcome` | type | The exact result of one parent→child delivery attempt. | +| `DriveHarness` | type | How to run an external harness as the DRIVER, with the coordination verbs mounted — the substrate | | `Environment` | type | A checkable task domain — implement these 5 hooks and the suite does the rest. The | | `EqualKOnCost` | type | `equalKOnCost(arms, opts)` — the cross-arm equal-compute check on conserved cost. | +| `ExecutionBindingReceipt` | type | One attempt's immutable link from a stable materialization plan to its actual transport. | | `ExecutorConfig` | type | Config for {@link createExecutor}: the backend is DATA — the cost dial a profile, | | `ExecutorFactory` | type | Builds a fresh `Executor` for one spawn from the resolved spec. Per-spawn (not | | `Fanout` | type | `fanout(items, opts)` — build the fanout combinator over a static item list. | @@ -904,16 +933,21 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `LoopOptionsForDispatch` | type | runAgentRounds options minus the `ctx` (loopDispatch builds the ctx). | | `LoopShape` | type | A reusable act-body factory. Given the persona's content + seams (`ShapeContext`), it | | `LoopUntil` | type | `loopUntil(spec)` — build the iterative-deepening combinator. `seed` is the initial state. | +| `MaterializedModelIdentity` | type | A named model carried into an execution, or an explicit reason the exact model is unknowable. | | `MountRecorder` | type | Records a mounted resource into the run's provenance manifest. Passed to | | `NodeId` | type | Deterministic node id — `${parent}:s${seq}` from the cursor order, never wall-clock. | | `NodeStatus` | type | `'acquiring'` is first-class (M1): a node spends real time + reaps an orphan box | +| `ObserveSupervisorNodeEvent` | type | Context-aware observer used internally to bind product transactions to the actual live node. | | `OpenSandboxRunPromptOptions` | type | Prompt options forwarded to every sandbox prompt turn in this run. The | | `Outcome` | type | The terminal contract Drew wants: a loop returns a FINISHED deliverable, or the concrete | | `Panel` | type | `panel(spec)` — build the M-judge write-only-merge combinator. | | `Pipeline` | type | `pipeline(stages)` — build the sequential combinator from an ordered stage list. The first | | `ProfileKeyOf` | type | The profile (matrix row) a record belongs to — default `harness·model` from the record's profile cell, | +| `ProfileMaterializationReceipt` | type | What the kernel can prove about one node's actual execution plan. | | `RenderCorpusToInstructions` | type | `renderCorpusToInstructions(opts)` — the flywheel read-back projection. Async (queries the | +| `ResolveSupervisorTools` | type | Product policy for the tools one exact supervisor node may call. Resolved once per node. | | `Restart` | type | OTP child-spec restart class. | +| `RootMaterialization` | type | Trusted root composition evidence. Generic `Agent.act` roots omit this and remain unknown. | | `RootSignal` | type | Out-of-band message to a running root. Open by intent — a client extends it. | | `RunContext` | type | The stores a supervised run needs, in-memory or file-backed. `InMemoryRunContext` is the | | `RunLoopOptions` | type | Pre-rename name for {@link RunAgentRoundsOptions}. | @@ -924,7 +958,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `Shell` | type | Command runner seam. Host code can use `localShell`; sandbox code can wrap `box.exec`. | | `SpawnEvent` | type | Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO | | `SpawnPrior` | type | What a KEYED spawn resolved to when the key had a prior attempt. Absent on a fresh key (and on | -| `SpawnRejection` | type | Fail-closed spawn rejections: an exhausted pool, an exceeded recursion ceiling, or a `key` | +| `SpawnRejection` | type | Fail-closed spawn rejections: an exhausted pool, an exceeded recursion ceiling, a full | | `SteeringDecision` | type | Terminal-or-continue decision shared by all three steering drivers. The | | `StopDecision` | type | A stop rule's answer. `reason` is required when stopping — a run that ends must be able to say | | `StopRule` | type | Evaluated from the progress feed, never from the budget. Pure and synchronous: it is called on | @@ -932,10 +966,13 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `StructuralRolloutMessage` | type | Provider-neutral conversation records read by structural candidate extraction. | | `SupervisedResult` | type | Typed terminal result (M2) — a no-winner is NEVER coerced to a best-effort output. | | `SupervisorFinalizer` | type | The finalization seam: ledger in, output (or `undefined` = nothing deliverable) out. | +| `SupervisorNodeContextSeed` | type | Context known before `Agent.act`; Runtime adds the concrete node, profile, and task. | +| `SupervisorProfile` | type | A supervisor is an ordinary, complete `AgentProfile` playing the supervisor role. | | `ToolLoopChat` | type | One inference turn over the running conversation + the tool specs → the model's text, any | | `ToolLoopCompactionOptions` | type | Public supervisor-facing compaction config: same knobs as the primitive, but `distill` is optional | | `ToolLoopMessageRecord` | type | Provider-neutral conversation record accepted by a tool-loop brain. | | `TrajectoryReportFn` | type | `trajectoryReport(...)` — the tree+cost reconstructor. Async (reads journal + optionally blobs). | +| `UnknownMaterializationReason` | type | Why exact materialization evidence is unavailable for a node. | | `UsageEvent` | type | Normalized usage event — the single channel every executor reports through, so the | | `Verify` | type | `verify(spec)` — build the 2-node implement→verifier-gate combinator. | | `WaitProbe` | type | A named predicate a `poll` node re-checks. Returns true when the condition it watches has | @@ -1173,7 +1210,7 @@ Import from `@tangle-network/agent-runtime/testing` — 4 exports. ### MCP servers — delegate / coordination / detached-session -Import from `@tangle-network/agent-runtime/mcp` — 206 exports. +Import from `@tangle-network/agent-runtime/mcp` — 213 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1260,11 +1297,13 @@ Import from `@tangle-network/agent-runtime/mcp` — 206 exports. | `InMemoryFeedbackStore` | class | In-memory `FeedbackStore` — suitable for single-process use and tests. | | `AgentMemorySpec` | interface | The `memory` artifact payload — HOW a profile's memory is stored and served: | | `AnalystFindingEvent` | interface | A trace-analyst result re-entered as a message on the bus (the `finding` event kind). | +| `AuthorizedDownMessage` | interface | Product-authorized continuation bytes. Returning a narrowed instruction replaces the proposed | | `Check` | interface | One lens — a composable analyst kind. Identity fields mirror `TraceAnalystKindSpec` so a kind is | | `CodexExecutionEvidence` | interface | Zero-model-call evidence for the exact Codex process about to run. | | `CodexExecutionFailureDiagnostic` | interface | Bounded, credential-redacted process context attached when reproducible Codex output fails | | `CodexExecutionPolicy` | interface | Isolation settings asserted before a reproducible Codex run is allowed to start. | | `CodexTokenUsage` | interface | Exact aggregate usage emitted by Codex's terminal `turn.completed` JSONL event. | +| `ContinuationInstruction` | interface | Durable authorization receipt written before a continuation reaches a worker. | | `CoordinationTools` | interface | The supervisor-side toolbox returned by {@link createCoordinationTools}: the MCP tool | | `DelegateArgs` | interface | Parsed `delegate` tool arguments. | | `DelegateCodeConfig` | interface | Minimal `CoderTask` overrides exposed over the MCP wire. The full | @@ -1274,7 +1313,9 @@ Import from `@tangle-network/agent-runtime/mcp` — 206 exports. | `DelegationTraceCollector` | interface | Per-delegation trace collector. Buffers `LoopTraceEvent`s per runId | | `DelegationTraceSpan` | interface | One span of a delegation's compact trace. Flat (parent linkage by id), all | | `DetachedSessionRefParts` | interface | Decoded `DelegationRecord.detachedSessionRef`. `sandboxId` is absent between | -| `DownMessageEvent` | interface | A parent→child message (the down-leg): recorded for observability, delivered via the child inbox, | +| `DownMessageAuthorizationInput` | interface | Detached continuation bytes and exact worker identity presented to product authorization before | +| `DownMessageDeliveryAttempt` | interface | A durable marker written after authorization and immediately before Runtime calls `Scope.send`. | +| `DownMessageEvent` | interface | A parent→child delivery result (the down-leg): recorded for observability, never pulled back by | | `DriveTurnCapableBox` | interface | The box surface detached turns need. `SandboxInstance` | | `FleetHandle` | interface | Minimal `SandboxFleet` surface the fleet executor calls. Declared | | `JsonRpcMessage` | interface | One JSON-RPC 2.0 request or notification. | @@ -1286,13 +1327,16 @@ Import from `@tangle-network/agent-runtime/mcp` — 206 exports. | `ResolvedMemoryEnv` | interface | What the memory bin resolved from its environment. | | `SettledWorker` | interface | A worker the driver has drained via `await_event`. | | `UiAuditorDelegationOutput` | interface | Wire-shape of a completed UI-audit delegation. The `findings` array | +| `WorkerSpawnContext` | interface | Immutable task, allocation, identity attribution, and semantic key supplied while a manager's | | `WorkerWatchOptions` | interface | Online-detector wiring for spawned workers (`CoordinationToolsOptions.watchWorkers`). | +| `AuthorizeDownMessage` | type | Product decision over an exact continuation before it is durably recorded or delivered. | | `CoderReviewer` | type | Optional adversarial reviewer over a coder candidate that already passed | | `CoordinationEvent` | type | Every message on the one typed pipe. UP (child→parent): question / settled / finding — queued for | | `DelegateResult` | type | The synchronous result the `delegate` tool returns to the calling agent: the delivered output (or | | `DelegationArgs` | type | Arguments accepted by the durable delegation queue. | | `DelegationResultPayload` | type | Polymorphic `result` field: `CoderOutput` when the underlying profile | | `DelegationResumeTick` | type | One observation of a detached run, mapped 1:1 from a single-tick driver | +| `DownMessageDeliveryOutcome` | type | The exact result of one parent→child delivery attempt. | | `DriveTurnTick` | type | Structural mirror of the sandbox SDK's `TurnDriveResult` (>= 0.6). | | `GitRunner` | type | Pluggable git runner (sync) — replaceable in tests. | | `LocalHarness` | type | Local coding harness available inside the sandbox. | diff --git a/src/durable/spawn-journal.ts b/src/durable/spawn-journal.ts index 0d2274e3..3fe71f7d 100644 --- a/src/durable/spawn-journal.ts +++ b/src/durable/spawn-journal.ts @@ -22,6 +22,7 @@ import { createHash } from 'node:crypto' import type { + NodeExecutionIdentity, NodeId, NodeSnapshot, NodeStatus, @@ -35,6 +36,7 @@ import type { } from '../runtime/supervise/types' import type { PendingWait } from '../runtime/supervise/wait' import { zeroTokenUsage } from '../runtime/util' +import { parseCommittedJsonLines, prepareJsonlAppend, writeAllBytes } from './jsonl-file' // ── Content addressing ────────────────────────────────────────────────────── @@ -178,6 +180,8 @@ export class InMemorySpawnJournal implements SpawnJournal { * writes never loses an acknowledged event. */ export class FileSpawnJournal implements SpawnJournal { + private appendTail: Promise = Promise.resolve() + constructor(private readonly path: string) {} async loadTree(root: NodeId): Promise { @@ -189,11 +193,9 @@ export class FileSpawnJournal implements SpawnJournal { if (isNoEntError(err)) return undefined throw err } - const lines = text.split('\n').filter((line) => line.length > 0) let begun = false const events: SpawnEvent[] = [] - for (const line of lines) { - const record = JSON.parse(line) as SpawnJournalRecord + for (const record of parseCommittedJsonLines(text, this.path)) { if (record.root !== root) continue if (record.kind === 'begin') { begun = true @@ -241,21 +243,26 @@ export class FileSpawnJournal implements SpawnJournal { if (isNoEntError(err)) return undefined throw err } - const lines = text.split('\n').filter((line) => line.length > 0) - for (const line of lines) { - const record = JSON.parse(line) as SpawnJournalRecord + for (const record of parseCommittedJsonLines(text, this.path)) { if (record.root === root && record.kind === 'begin') return record.at } return undefined } private async appendRecord(record: SpawnJournalRecord): Promise { + const append = this.appendTail.then(() => this.writeRecord(record)) + this.appendTail = append.catch(() => undefined) + return append + } + + private async writeRecord(record: SpawnJournalRecord): Promise { const fs = await import('node:fs/promises') const path = await import('node:path') await fs.mkdir(path.dirname(this.path), { recursive: true }) + const needsSeparator = await prepareJsonlAppend(this.path) const fh = await fs.open(this.path, 'a') try { - await fh.write(`${JSON.stringify(record)}\n`) + await writeAllBytes(fh, `${needsSeparator ? '\n' : ''}${JSON.stringify(record)}\n`) await fh.sync() } finally { await fh.close() @@ -276,6 +283,37 @@ type SpawnJournalRecord = * ordinal legitimately equals a later `settled` cursor seq and is not a collision. */ function assertSeqUnique(root: NodeId, events: SpawnEvent[], ev: SpawnEvent): void { + if (ev.kind === 'materialized') { + if (events.some((event) => event.kind === 'materialized' && event.id === ev.id)) { + throw new Error( + `spawn journal corrupted: duplicate materialization receipt for node '${ev.id}' in tree '${root}'`, + ) + } + if (!events.some((event) => event.kind === 'spawned' && event.id === ev.id)) { + throw new Error( + `spawn journal corrupted: materialization for node '${ev.id}' precedes its spawn in tree '${root}'`, + ) + } + } + if (ev.kind === 'execution-bound') { + if ( + events.some( + (event) => + event.kind === 'execution-bound' && + event.id === ev.id && + event.binding.attemptId === ev.binding.attemptId, + ) + ) { + throw new Error( + `spawn journal corrupted: duplicate execution binding for node '${ev.id}' attempt '${ev.binding.attemptId}' in tree '${root}'`, + ) + } + if (!events.some((event) => event.kind === 'materialized' && event.id === ev.id)) { + throw new Error( + `spawn journal corrupted: execution binding for node '${ev.id}' precedes materialization in tree '${root}'`, + ) + } + } // `spawned` (ordinal namespace), `waiting` (the wait-ordinal namespace — it CREATES a node, it // does not settle one), and `metered` (informational spend, no settlement order) live outside // the cursor-uniqueness namespace replay relies on. `woken` IS a settlement and does not. @@ -292,7 +330,13 @@ function assertSeqUnique(root: NodeId, events: SpawnEvent[], ev: SpawnEvent): vo * ordering rests on. The single predicate both the guard's sides read, so a new event kind is * classified once. */ function outsideCursorNamespace(ev: SpawnEvent): boolean { - return ev.kind === 'spawned' || ev.kind === 'waiting' || ev.kind === 'metered' + return ( + ev.kind === 'spawned' || + ev.kind === 'waiting' || + ev.kind === 'metered' || + ev.kind === 'materialized' || + ev.kind === 'execution-bound' + ) } // ── Replay executor (build step 7) ─────────────────────────────────────────────── @@ -319,24 +363,57 @@ export async function replaySpawnTree( } const ordered = [...events].sort((a, b) => a.seq - b.seq) const labels = new Map() + const assignmentIds = new Map() + const identities = new Map() + const materializations = new Map() + const executionBindings = new Map< + NodeId, + NonNullable[number][] + >() for (const ev of ordered) { if (ev.kind === 'spawned' || ev.kind === 'waiting') labels.set(ev.id, ev.label) + if (ev.kind === 'spawned' && ev.assignmentId !== undefined) { + assignmentIds.set(ev.id, ev.assignmentId) + } + if (ev.kind === 'spawned' && ev.identity !== undefined) { + identities.set(ev.id, copyFrozenIdentity(ev.identity)) + } + if (ev.kind === 'materialized') materializations.set(ev.id, ev.receipt) + if (ev.kind === 'execution-bound') { + const bindings = executionBindings.get(ev.id) ?? [] + bindings.push(ev.binding) + executionBindings.set(ev.id, bindings) + } + } + const handleFor = (id: NodeId, status: NodeStatus) => + replayHandle(id, labels.get(id) ?? id, status, { + assignmentId: assignmentIds.get(id), + identity: identities.get(id), + materialization: materializations.get(id), + executionBindings: executionBindings.get(id), + }) + const settlementTime = (at: string): { readonly settledAt?: number } => { + const settledAt = Date.parse(at) + return Number.isFinite(settledAt) ? { settledAt } : {} } const settled: Settled[] = [] for (const ev of ordered) { if (ev.kind === 'spawned') continue if (ev.kind === 'waiting') continue // arms a wait node; `woken` is its settlement if (ev.kind === 'metered') continue // a spend record, not a settlement — irrelevant to replay + if (ev.kind === 'materialized') continue // wire receipt, not a settlement + if (ev.kind === 'execution-bound') continue // attempt transport, not a settlement if (ev.kind === 'woken') { // A wait that was cancelled carries no outcome blob — it replays as a `down`, exactly as a // cancelled worker does. A fired/timed-out wait rehydrates its `WaitOutcome` and costs zero. if (ev.by === 'cancelled' || ev.outRef === undefined) { settled.push({ kind: 'down', - handle: replayHandle(ev.id, labels.get(ev.id) ?? ev.id, 'cancelled'), + handle: handleFor(ev.id, 'cancelled'), reason: 'wait cancelled', infra: false, restartCount: 0, + ...settlementTime(ev.at), seq: ev.seq, }) continue @@ -349,10 +426,11 @@ export async function replaySpawnTree( } settled.push({ kind: 'done', - handle: replayHandle(ev.id, labels.get(ev.id) ?? ev.id, 'done'), + handle: handleFor(ev.id, 'done'), out: outcome, outRef: ev.outRef, spent: zeroSpend(), + ...settlementTime(ev.at), seq: ev.seq, }) continue @@ -360,10 +438,11 @@ export async function replaySpawnTree( if (ev.kind === 'cancelled') { settled.push({ kind: 'down', - handle: replayHandle(ev.id, labels.get(ev.id) ?? ev.id, 'cancelled'), + handle: handleFor(ev.id, 'cancelled'), reason: ev.reason, infra: false, restartCount: 0, + ...settlementTime(ev.at), seq: ev.seq, }) continue @@ -371,10 +450,11 @@ export async function replaySpawnTree( if (ev.status === 'down') { settled.push({ kind: 'down', - handle: replayHandle(ev.id, labels.get(ev.id) ?? ev.id, 'failed'), + handle: handleFor(ev.id, 'failed'), reason: ev.verdict?.notes ?? 'child down', infra: ev.infra === true, restartCount: 0, + ...settlementTime(ev.at), seq: ev.seq, }) continue @@ -393,26 +473,61 @@ export async function replaySpawnTree( } settled.push({ kind: 'done', - handle: replayHandle(ev.id, labels.get(ev.id) ?? ev.id, 'done'), + handle: handleFor(ev.id, 'done'), out, outRef: ev.outRef, verdict: ev.verdict, spent: ev.spent, + ...settlementTime(ev.at), seq: ev.seq, }) } return settled } -function replayHandle(id: NodeId, label: string, status: NodeStatus) { - return { +function replayHandle( + id: NodeId, + label: string, + status: NodeStatus, + evidence: { + readonly assignmentId?: string + readonly identity?: NodeExecutionIdentity + readonly materialization?: NodeSnapshot['materialization'] + readonly executionBindings?: ReadonlyArray< + NonNullable[number] + > + } = {}, +) { + return Object.freeze({ id, label, status, + ...(evidence.assignmentId === undefined ? {} : { assignmentId: evidence.assignmentId }), + ...(evidence.identity === undefined ? {} : { identity: evidence.identity }), + ...(evidence.materialization === undefined + ? {} + : { materialization: evidence.materialization }), + ...(evidence.executionBindings === undefined + ? {} + : { executionBindings: Object.freeze([...evidence.executionBindings]) }), abort() { throw new Error(`cannot abort node '${id}': replayed handles are terminal, not live`) }, - } + }) +} + +/** A replayed handle must not retain mutable references into the loaded journal event. */ +function copyFrozenIdentity(identity: NodeExecutionIdentity): NodeExecutionIdentity { + const correlation = + identity.correlation === undefined ? undefined : Object.freeze({ ...identity.correlation }) + return Object.freeze({ + ...(identity.profileDigest === undefined ? {} : { profileDigest: identity.profileDigest }), + ...(identity.taskDigest === undefined ? {} : { taskDigest: identity.taskDigest }), + ...(identity.candidateDigest === undefined + ? {} + : { candidateDigest: identity.candidateDigest }), + ...(correlation === undefined ? {} : { correlation }), + }) } /** @@ -435,7 +550,14 @@ export function materializeTreeView(events: SpawnEvent[]): TreeView { ) .sort((a, b) => a.seq - b.seq) const settlements = events - .filter((ev) => ev.kind !== 'spawned' && ev.kind !== 'waiting' && ev.kind !== 'metered') + .filter( + (ev) => + ev.kind !== 'spawned' && + ev.kind !== 'waiting' && + ev.kind !== 'metered' && + ev.kind !== 'materialized' && + ev.kind !== 'execution-bound', + ) .sort((a, b) => a.seq - b.seq) for (const ev of spawns) { if (ev.kind === 'waiting') { @@ -460,6 +582,8 @@ export function materializeTreeView(events: SpawnEvent[]): TreeView { status: 'pending', runtime: ev.runtime, budget: ev.budget, + ...(ev.assignmentId === undefined ? {} : { assignmentId: ev.assignmentId }), + ...(ev.identity ? { identity: ev.identity } : {}), spent: zeroSpend(), }) } @@ -469,15 +593,35 @@ export function materializeTreeView(events: SpawnEvent[]): TreeView { node.status = ev.status === 'done' ? 'done' : 'failed' node.spent = ev.spent node.outRef = ev.outRef + const settledAt = Date.parse(ev.at) + if (Number.isFinite(settledAt)) node.settledAt = settledAt } else if (ev.kind === 'woken') { const node = requireNode(nodes, ev.id) node.status = ev.by === 'cancelled' ? 'cancelled' : 'done' node.outRef = ev.outRef + const settledAt = Date.parse(ev.at) + if (Number.isFinite(settledAt)) node.settledAt = settledAt } else { const node = requireNode(nodes, ev.id) node.status = 'cancelled' + const settledAt = Date.parse(ev.at) + if (Number.isFinite(settledAt)) node.settledAt = settledAt } } + // Materialization is node evidence, not a settlement. Fold it after node creation and before + // freezing the view; exactly one receipt per node is enforced by the journal corruption guard. + for (const ev of events) { + if (ev.kind !== 'materialized') continue + const node = requireNode(nodes, ev.id) + node.materialization = ev.receipt + node.runtime = ev.receipt.runtime + } + for (const ev of events) { + if (ev.kind !== 'execution-bound') continue + const node = requireNode(nodes, ev.id) + node.executionBindings ??= [] + node.executionBindings.push(ev.binding) + } // Driver inference: a separate pass so it accumulates ONTO the settled child-work base (no // dependence on metered-vs-settled seq order) without touching node status. for (const ev of events) { @@ -523,8 +667,13 @@ interface MutableSnapshot { status: NodeStatus runtime: Runtime budget: NodeSnapshot['budget'] + assignmentId?: string + identity?: NodeSnapshot['identity'] + materialization?: NodeSnapshot['materialization'] + executionBindings?: NonNullable[number][] spent: Spend outRef?: string + settledAt?: number } function zeroSpend(): Spend { @@ -536,6 +685,7 @@ function addJournalSpend(a: Spend, b: Spend): Spend { return { iterations: a.iterations + b.iterations, tokens: { input: a.tokens.input + b.tokens.input, output: a.tokens.output + b.tokens.output }, + ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), usd: a.usd + b.usd, ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), ms: a.ms + b.ms, @@ -558,8 +708,14 @@ function freezeSnapshot(node: MutableSnapshot): NodeSnapshot { status: node.status, runtime: node.runtime, budget: node.budget, + assignmentId: node.assignmentId, + identity: node.identity, + materialization: node.materialization, + executionBindings: + node.executionBindings === undefined ? undefined : Object.freeze([...node.executionBindings]), spent: node.spent, outRef: node.outRef, + settledAt: node.settledAt, } } diff --git a/src/mcp/index.ts b/src/mcp/index.ts index 89b4826a..8bc044f9 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -159,11 +159,17 @@ export { export { type AnalystFindingEvent, type AnalystRegistry, + type AuthorizeDownMessage, + type AuthorizedDownMessage, + type ContinuationInstruction, type CoordinationEvent, type CoordinationTools, type CoordinationToolsOptions, createCoordinationTools, DEFAULT_AWAIT_EVENT_TIMEOUT_MS, + type DownMessageAuthorizationInput, + type DownMessageDeliveryAttempt, + type DownMessageDeliveryOutcome, type DownMessageEvent, type MakeWorkerAgent, type Question, @@ -174,6 +180,7 @@ export { type QuestionRecord, type QuestionUrgency, type SettledWorker, + type WorkerSpawnContext, type WorkerWatchOptions, } from './tools/coordination' export { diff --git a/src/mcp/tools/coordination.ts b/src/mcp/tools/coordination.ts index 040fffea..714220b5 100644 --- a/src/mcp/tools/coordination.ts +++ b/src/mcp/tools/coordination.ts @@ -8,13 +8,25 @@ * @experimental */ +import { randomUUID } from 'node:crypto' +import { + type AgentProfile, + agentProfileSchema, + canonicalCandidateDigest, +} from '@tangle-network/agent-interface' import type { + AgentExecutionRef, Budget, + ExecutionBindingReceipt, + NodeExecutionIdentity, + ProfileMaterializationReceipt, ResultBlobStore, Scope, Settled, + Spend, Agent as SuperviseAgent, } from '../../runtime' +import { assertValidBudget } from '../../runtime/supervise/budget' import { type WatchTraceOptions, watchTrace } from '../../runtime/supervise/detector-monitor' import { freeSlots } from '../../runtime/supervise/dispatch' import { type BusRecord, type BusStats, createEventBus } from '../../runtime/supervise/event-bus' @@ -25,13 +37,24 @@ import type { McpToolDescriptor } from '../server' export interface SettledWorker { readonly id: string readonly status: 'done' | 'down' + /** Stable manager-scoped assignment, including deterministic unkeyed siblings. */ + readonly assignmentId?: string + /** Exact profile/task/candidate identity authorized for this node. */ + readonly identity?: NodeExecutionIdentity + /** Stable effective execution plan, or an explicit unknown receipt. */ + readonly materialization?: ProfileMaterializationReceipt + /** Backend bindings for each attempt, in durable oldest-first order. */ + readonly executionBindings?: ReadonlyArray + /** Conserved spend. Missing means unavailable; unknown accounting remains explicitly unknown. */ + readonly spent?: Spend readonly score?: number readonly valid?: boolean readonly outRef?: string readonly reason?: string - /** Epoch ms the ledger recorded this settlement — the resolution a progress-based stop rule - * needs to answer "how long since anything landed?" without inventing a timestamp at read - * time. Stamped when the cursor yields the settlement, not when a reader first looks. */ + /** True when projected from a prior process of the same durable run. */ + readonly resumed?: boolean + /** Epoch ms from the durable terminal record — the resolution a progress-based stop rule needs + * to answer "how long since anything landed?" without inventing a timestamp at read time. */ readonly settledAt?: number } @@ -79,25 +102,110 @@ export interface AnalystFindingEvent { readonly findings: unknown } -/** A parent→child message (the down-leg): recorded for observability, delivered via the child inbox, - * never pulled back by the parent. `delivered` mirrors whether the live child accepted it. */ +/** The exact result of one parent→child delivery attempt. */ +export type DownMessageDeliveryOutcome = + | 'delivered' + | 'unknown-worker' + | 'already-settled' + | 'runtime-has-no-inbox' + | 'scope-stopped' + | 'runtime-error' + +/** A durable marker written after authorization and immediately before Runtime calls `Scope.send`. + * If a process dies with this marker but no matching outcome, delivery is unknown and is never + * replayed automatically. */ +export interface DownMessageDeliveryAttempt { + readonly receiptId: string + readonly kind: 'steer' | 'answer' + readonly toWorker: string + readonly instructionDigest: string + readonly interrupt: boolean + readonly questionId?: string +} + +/** A parent→child delivery result (the down-leg): recorded for observability, never pulled back by + * the parent. `receiptId` and `instructionDigest` link it to the pre-delivery authorization receipt + * and attempt marker. */ export interface DownMessageEvent { + readonly receiptId: string readonly toWorker: string readonly instruction: string + readonly instructionDigest: string readonly delivered: boolean + readonly outcome: DownMessageDeliveryOutcome + readonly error?: string +} + +/** Durable authorization receipt written before a continuation reaches a worker. */ +export interface ContinuationInstruction { + readonly receiptId: string + readonly kind: 'steer' | 'answer' + readonly toWorker: string + readonly instruction: string + readonly instructionDigest: string + readonly workerIdentity?: NodeExecutionIdentity + readonly interrupt: boolean + readonly questionId?: string +} + +/** Detached continuation bytes and exact worker identity presented to product authorization before + * Runtime records or delivers a steer/answer. */ +export interface DownMessageAuthorizationInput { + readonly kind: 'steer' | 'answer' + readonly workerId: string + readonly workerIdentity: NodeExecutionIdentity + readonly instruction: string + readonly interrupt: boolean + readonly questionId?: string +} + +/** Product-authorized continuation bytes. Returning a narrowed instruction replaces the proposed + * bytes; throwing refuses delivery. */ +export interface AuthorizedDownMessage { + readonly instruction: string } +/** Product decision over an exact continuation before it is durably recorded or delivered. */ +export type AuthorizeDownMessage = (input: DownMessageAuthorizationInput) => AuthorizedDownMessage + /** Every message on the one typed pipe. UP (child→parent): question / settled / finding — queued for - * the driver to `pull`. DOWN (parent→child): steer / answer — record-only (history + subscribers), - * routed to the child inbox. New kinds are additive. */ + * the driver to `pull`. An `instruction` is the pre-delivery authorization receipt and is retained + * as evidence. DOWN (parent→child): steer / answer — record-only (history + subscribers), routed + * to the child inbox. Receipts are never auto-delivered on restart. New kinds are additive. */ export type CoordinationEvent = | { readonly type: 'question'; readonly question: QuestionRecord } | { readonly type: 'settled'; readonly worker: SettledWorker } | { readonly type: 'finding'; readonly finding: AnalystFindingEvent } | { readonly type: 'steer'; readonly down: DownMessageEvent } | { readonly type: 'answer'; readonly down: DownMessageEvent; readonly questionId: string } + | { readonly type: 'instruction'; readonly instruction: ContinuationInstruction } + | { readonly type: 'delivery-attempt'; readonly attempt: DownMessageDeliveryAttempt } + +/** Immutable task, allocation, identity attribution, and semantic key supplied while a manager's + * complete worker profile is prepared for one spawn. */ +export interface WorkerSpawnContext { + /** Stable assignment identity within this manager. A semantic key wins; otherwise Runtime mints + * the manager's deterministic pre-factory spawn ordinal so identical unkeyed siblings stay + * isolated and can recover by issuing the same assignments in the same order. */ + readonly assignmentId: string + /** Trusted concrete manager node authorizing this spawn. Never accepted from model arguments. */ + readonly parentNodeId: string + /** The exact allocation this node receives after the tool's optional override is merged. */ + readonly budget: Budget + /** Detached, deeply immutable task bytes from this spawn request. */ + readonly task: unknown + /** Exact trace label selected for this spawn. */ + readonly label: string + /** Semantic restart key, when the manager supplied one. */ + readonly key?: string + /** Trusted candidate/campaign attribution attached by product authorization. */ + readonly execution?: AgentExecutionRef +} -export type MakeWorkerAgent = (profile: unknown) => SuperviseAgent +export type MakeWorkerAgent = ( + profile: AgentProfile, + context?: WorkerSpawnContext, +) => SuperviseAgent export interface CoordinationToolsOptions { readonly scope: Scope @@ -105,7 +213,17 @@ export interface CoordinationToolsOptions { readonly makeWorkerAgent: MakeWorkerAgent readonly perWorker: Budget readonly analysts?: AnalystRegistry - readonly onEvent?: (event: CoordinationEvent) => void | Promise + /** Event-first for source compatibility; the second argument is its exact bus ordering stamp. */ + readonly onEvent?: ( + event: CoordinationEvent, + record: BusRecord, + ) => void | Promise + /** Re-publish resumed settlements through the awaited observer before the driver starts. This is + * the crash-window recovery path for product transactions; off preserves low-level legacy reads. */ + readonly replaySettlements?: boolean + /** Authorize each continuation against the exact worker identity. The returned instruction is + * detached, recorded durably through `onEvent`, and only then delivered. */ + readonly authorizeDownMessage?: AuthorizeDownMessage readonly questionPolicy?: QuestionPolicy /** Analyst kind ids to run AUTOMATICALLY when a worker settles `done` (the analyst-on-settle * hook). Each result is published as a `finding` event on the bus — pass-through to subscribers @@ -116,7 +234,8 @@ export interface CoordinationToolsOptions { * counts the scope's non-terminal nodes and fails closed (`error: 'max-live-workers'`) BEFORE * reserving from the pool when the cap is already met — a concurrency fence on top of the * conserved-budget fence (the pool bounds total work; this bounds simultaneous work, e.g. live - * sandboxes/boxes). Omit or `<= 0` = no cap (the prior behavior; the pool stays the only fence). */ + * sandboxes/boxes). A tree-wide limit owned by `Scope` takes precedence when present; this field + * is the local form for a caller-owned scope. Omit or `<= 0` = no local cap. */ readonly maxLiveWorkers?: number /** Max wall-clock ms a single `await_event` call may block waiting on a live worker to settle * before it returns a non-error `{ pending: true, live }` snapshot and lets the caller re-poll. @@ -179,13 +298,15 @@ export const DEFAULT_AWAIT_EVENT_TIMEOUT_MS = 15_000 */ export interface CoordinationTools { readonly tools: McpToolDescriptor[] + /** Commit any resume-time event replay before a supervisor can reason or an MCP can listen. */ + ready(): Promise isStopped(): boolean stopReason(): string | undefined settled(): ReadonlyArray questions(): ReadonlyArray - /** The full ordered log of every bus event — UP (settled / question / finding) and DOWN - * (steer / answer) — the observability audit + replay trail. Each record carries seq, - * timestamp, and priority. */ + /** The full ordered log of every bus event — UP (settled / question / finding), authorized + * instruction receipts, and DOWN delivery outcomes (steer / answer). Each record carries seq, + * timestamp, and priority. A receipt is evidence and is never auto-delivered on restart. */ history(): ReadonlyArray> /** Bus throughput counters (published / pulled / by-kind) for live dashboards. */ stats(): BusStats @@ -238,35 +359,90 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin // slot, so the fence must not hold it back. `keyByWorker` is what lets a settlement find its key. const completedKeys = new Set() const keyByWorker = new Map() + let unkeyedAssignmentOrdinal = 0 for (const [key, prior] of opts.scope.resume?.keys ?? []) { if (prior.state === 'completed') completedKeys.add(key) } - // A resumed scope's replayed settlements enter the ledger AT CONSTRUCTION, so `settled()` — and - // therefore the finalize that reads it — spans processes exactly as the journal does. They are - // NOT re-published on the bus: the driver's resume brief already lists them, and the scope - // cursor only yields THIS process's children, so nothing double-counts. - for (const s of opts.scope.resume?.settled ?? []) { - ledger.push( - s.kind === 'done' + const nodeForWorker = (id: string) => + opts.scope.view.nodes.find((node) => node.id === id) ?? + opts.scope.resume?.view.nodes.find((node) => node.id === id) + + const projectSettled = (settled: Settled, resumed = false): SettledWorker => { + const node = nodeForWorker(settled.handle.id) + const assignmentId = settled.handle.assignmentId ?? node?.assignmentId + const identity = settled.handle.identity ?? node?.identity + const materialization = settled.handle.materialization ?? node?.materialization + const executionBindings = settled.handle.executionBindings ?? node?.executionBindings + const settledAt = settled.settledAt ?? node?.settledAt + const common = { + id: settled.handle.id, + ...(assignmentId === undefined ? {} : { assignmentId }), + ...(identity === undefined ? {} : { identity }), + ...(materialization === undefined ? {} : { materialization }), + ...(executionBindings === undefined ? {} : { executionBindings }), + ...(settledAt === undefined ? {} : { settledAt }), + ...(resumed ? { resumed: true as const } : {}), + } + return deepFreezeDetached( + settled.kind === 'done' ? { - id: s.handle.id, + ...common, status: 'done', - score: s.verdict?.score ?? 0, - valid: s.verdict?.valid ?? false, - outRef: s.outRef, + spent: settled.spent, + ...(settled.verdict?.score === undefined ? {} : { score: settled.verdict.score }), + ...(settled.verdict?.valid === undefined ? {} : { valid: settled.verdict.valid }), + outRef: settled.outRef, } - : { id: s.handle.id, status: 'down', reason: s.reason }, + : { + ...common, + status: 'down', + ...(node?.spent === undefined ? {} : { spent: node.spent }), + reason: settled.reason, + }, ) } - // The one child→parent pipe. `onEvent` (back-compat) becomes a pass-through subscriber receiving - // the bare event, so every kind — question, settled, finding — reaches it immediately, and the - // driver pulls queued findings / questions via `await_event`. + // A resumed scope's replayed settlements enter the ledger AT CONSTRUCTION, so `settled()` — and + // therefore the finalize that reads it — spans processes exactly as the journal does. + const resumedWorkers: SettledWorker[] = [] + for (const s of opts.scope.resume?.settled ?? []) { + const worker = projectSettled(s, true) + resumedWorkers.push(worker) + ledger.push(worker) + } + + // The one child→parent pipe. Keep the event-first callback and also pass its exact stamp, so a + // durable subscriber retains causal sequence, original timestamp, and priority. const bus = createEventBus() if (opts.onEvent) { const cb = opts.onEvent - bus.subscribe((rec) => cb(rec.event)) + bus.subscribe((rec) => cb(rec.event, rec)) + } + // A settlement can be durable in the spawn journal while the process dies before the product + // observer acknowledges it. Opted-in high-level callers replay those events at least once. Keep + // each frozen event object across an in-process retry so EventBus reuses its exact BusRecord. + const resumeEvents = opts.replaySettlements + ? resumedWorkers.map((worker) => + deepFreezeDetached({ type: 'settled', worker }), + ) + : [] + let resumeEventIndex = 0 + let readyInFlight: Promise | undefined + const ready = (): Promise => { + if (resumeEventIndex >= resumeEvents.length) return Promise.resolve() + if (readyInFlight) return readyInFlight + readyInFlight = (async () => { + while (resumeEventIndex < resumeEvents.length) { + const event = resumeEvents[resumeEventIndex] + if (!event) break + await bus.publish(event) + resumeEventIndex += 1 + } + })().finally(() => { + readyInFlight = undefined + }) + return readyInFlight } // Urgency → bus priority: a blocking question is bumped ahead of queued settles/findings so the @@ -303,7 +479,7 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin const maxTokens = field('maxTokens') const maxUsd = field('maxUsd') const deadlineMs = field('deadlineMs') - return { + const merged: Budget = { maxIterations: maxIterations ?? base.maxIterations, maxTokens: maxTokens ?? base.maxTokens, ...((maxUsd ?? base.maxUsd) === undefined ? {} : { maxUsd: maxUsd ?? base.maxUsd }), @@ -311,6 +487,8 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin ? {} : { deadlineMs: deadlineMs ?? base.deadlineMs }), } + assertValidBudget(merged, 'coordination tools: budget') + return merged } const level = (v: unknown): Question['level'] => { if (v === 'worker' || v === 'driver' || v === 'loop') return v @@ -323,28 +501,52 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin ) } - const recordSettled = (s: Settled): SettledWorker => { - const settledAt = Date.now() + const commitSettled = (s: Settled, w: SettledWorker): void => { // A keyed assignment that just delivered is complete for the rest of this run, so a later // spawn under the same key resolves for free instead of being held behind the live-worker // fence (it starts no worker, so it occupies no slot). const settledKey = keyByWorker.get(s.handle.id) if (settledKey !== undefined && s.kind === 'done') completedKeys.add(settledKey) - const w: SettledWorker = - s.kind === 'done' - ? { - id: s.handle.id, - status: 'done', - score: s.verdict?.score ?? 0, - valid: s.verdict?.valid ?? false, - outRef: s.outRef, - settledAt, - } - : { id: s.handle.id, status: 'down', reason: s.reason, settledAt } ledger.push(w) // A settled worker's trace source is finished; drop the online subscription with it. unwatchWorker(w.id) - return w + } + + // `Scope.next()` is once-only, while an awaited observer may commit and lose its acknowledgement. + // Retain the exact event until publication succeeds so the next await retries rather than losing + // the settlement between the spawn journal and the product transaction. + let pendingSettlement: + | { + readonly settled: Settled + readonly worker: SettledWorker + readonly event: CoordinationEvent + readonly analyze: boolean + } + | undefined + + const flushPendingSettlement = async (): Promise => { + const pending = pendingSettlement + if (!pending) return false + await bus.publish(pending.event) + commitSettled(pending.settled, pending.worker) + pendingSettlement = undefined + if ( + pending.analyze && + pending.worker.status === 'done' && + pending.worker.outRef && + opts.analysts && + opts.analyzeOnSettle?.length + ) { + const trace = await opts.blobs.get(pending.worker.outRef) + for (const analyst of opts.analyzeOnSettle) { + const findings = await opts.analysts.run(analyst, trace) + await bus.publish({ + type: 'finding', + finding: { fromWorker: pending.worker.id, analyst, findings }, + }) + } + } + return true } // Producer: drain exactly one settlement from the scope cursor onto the bus (a `settled` event), @@ -352,18 +554,18 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin // publish its result as a `finding`. Returns false when the cursor is idle (no live workers). The // cursor is a once-per-child source, so a settlement is produced at most once. const drainSettlement = async (): Promise => { - const s = await opts.scope.next() - if (!s) return false - const w = recordSettled(s) - await bus.publish({ type: 'settled', worker: w }) - if (w.status === 'done' && w.outRef && opts.analysts && opts.analyzeOnSettle?.length) { - const trace = await opts.blobs.get(w.outRef) - for (const analyst of opts.analyzeOnSettle) { - const findings = await opts.analysts.run(analyst, trace) - await bus.publish({ type: 'finding', finding: { fromWorker: w.id, analyst, findings } }) + if (!pendingSettlement) { + const settled = await opts.scope.next() + if (!settled) return false + const worker = projectSettled(settled) + pendingSettlement = { + settled, + worker, + event: deepFreezeDetached({ type: 'settled', worker }), + analyze: true, } } - return true + return flushPendingSettlement() } // Post-loop drain: every ALREADY-settled, unpulled child enters the ledger + audit trail. No @@ -372,10 +574,18 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin const drainResolved = async (): Promise => { let drained = 0 for (;;) { - const s = await opts.scope.nextResolved() - if (!s) return drained - const w = recordSettled(s) - await bus.publish({ type: 'settled', worker: w }) + if (!pendingSettlement) { + const settled = await opts.scope.nextResolved() + if (!settled) return drained + const worker = projectSettled(settled) + pendingSettlement = { + settled, + worker, + event: deepFreezeDetached({ type: 'settled', worker }), + analyze: false, + } + } + await flushPendingSettlement() drained += 1 } } @@ -398,30 +608,143 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin ) } + const authorizeInstruction = ( + kind: 'steer' | 'answer', + workerId: string, + instruction: string, + interrupt: boolean, + questionId?: string, + ): ContinuationInstruction => { + const workerIdentity = opts.scope.view.nodes.find((node) => node.id === workerId)?.identity + let authorizedInstruction = instruction + if (opts.authorizeDownMessage) { + if (workerIdentity === undefined) { + throw new Error( + `coordination tools: cannot authorize ${kind} for worker ${JSON.stringify(workerId)} without durable identity`, + ) + } + const decision = deepFreezeDetached( + opts.authorizeDownMessage( + deepFreezeDetached({ + kind, + workerId, + workerIdentity, + instruction, + interrupt, + ...(questionId !== undefined ? { questionId } : {}), + }), + ), + ) + if ( + typeof decision !== 'object' || + decision === null || + Array.isArray(decision) || + typeof decision.instruction !== 'string' || + decision.instruction.length === 0 + ) { + throw new Error('coordination tools: authorizeDownMessage must return an instruction') + } + authorizedInstruction = decision.instruction + } + return deepFreezeDetached({ + receiptId: randomUUID(), + kind, + toWorker: workerId, + instruction: authorizedInstruction, + instructionDigest: canonicalCandidateDigest(authorizedInstruction), + ...(workerIdentity !== undefined ? { workerIdentity } : {}), + interrupt, + ...(questionId !== undefined ? { questionId } : {}), + }) + } + + /** Publish before `scope.send`: an awaited durable subscriber therefore commits the exact bytes + * before the worker can observe them. */ + const recordInstruction = async (instruction: ContinuationInstruction): Promise => { + await bus.publish({ type: 'instruction', instruction }, { queue: false }) + } + + /** Commit delivery intent after the authorization receipt and before `Scope.send`. An attempt with + * no matching outcome after a crash is explicitly unknown and must never be replayed. */ + const recordDeliveryAttempt = async ( + instruction: ContinuationInstruction, + ): Promise => { + const attempt = deepFreezeDetached({ + receiptId: instruction.receiptId, + kind: instruction.kind, + toWorker: instruction.toWorker, + instructionDigest: instruction.instructionDigest, + interrupt: instruction.interrupt, + ...(instruction.questionId !== undefined ? { questionId: instruction.questionId } : {}), + }) + await bus.publish({ type: 'delivery-attempt', attempt }, { queue: false }) + return attempt + } + + const deliveryOutcome = (workerId: string, delivered: boolean): DownMessageDeliveryOutcome => { + if (delivered) return 'delivered' + if (opts.scope.signal.aborted) return 'scope-stopped' + const node = opts.scope.view.nodes.find((candidate) => candidate.id === workerId) + if (!node) return 'unknown-worker' + if (!isLive(node.status)) return 'already-settled' + return 'runtime-has-no-inbox' + } + + const attemptDelivery = async ( + instruction: ContinuationInstruction, + message: unknown, + ): Promise => { + await recordDeliveryAttempt(instruction) + let delivered = false + let outcome: DownMessageDeliveryOutcome + let error: string | undefined + try { + delivered = opts.scope.send(instruction.toWorker, message) + outcome = deliveryOutcome(instruction.toWorker, delivered) + } catch (cause) { + outcome = 'runtime-error' + error = cause instanceof Error ? cause.message : String(cause) + } + const down = deepFreezeDetached({ + receiptId: instruction.receiptId, + toWorker: instruction.toWorker, + instruction: instruction.instruction, + instructionDigest: instruction.instructionDigest, + delivered, + outcome, + ...(error !== undefined ? { error } : {}), + }) + if (instruction.kind === 'answer') { + await sendDown('answer', down, str(instruction.questionId, 'questionId')) + } else { + await sendDown('steer', down) + } + if (error !== undefined) throw new Error(`coordination tools: delivery failed: ${error}`) + return down + } + // Consumer projection: the wire shape the driver sees for a pulled bus event. const projectEvent = (ev: CoordinationEvent): Record => { if (ev.type === 'settled') { - const w = ev.worker - return w.status === 'done' - ? { - type: 'settled', - settled: w.id, - status: 'done', - score: w.score, - valid: w.valid, - outRef: w.outRef, - } - : { type: 'settled', settled: w.id, status: 'down', reason: w.reason } + const { id, status, ...evidence } = ev.worker + return { type: 'settled', settled: id, status, ...evidence } } if (ev.type === 'question') return { type: 'question', question: ev.question } if (ev.type === 'finding') return { type: 'finding', ...ev.finding } if (ev.type === 'answer') return { type: 'answer', ...ev.down, questionId: ev.questionId } + if (ev.type === 'instruction') return { type: 'instruction', ...ev.instruction } + if (ev.type === 'delivery-attempt') return { type: 'delivery-attempt', ...ev.attempt } // Down-leg `steer` is record-only (never queued), so the driver never pulls it; project // defensively for completeness. return { type: ev.type, ...ev.down } } - const nextQuestionId = (from: string): string => `${from}:q${questionSeq++}` + const nextQuestionId = (from: string): string => { + for (;;) { + const id = `${from}:q${questionSeq++}` + if (!questions.some((question) => question.id === id)) return id + } + } const normalizeQuestion = (q: QuestionInput, fallbackFrom: string): Question => { const from = str(q.from ?? fallbackFrom, 'from') return { @@ -499,21 +822,48 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin }) } - // Count workers that are LIVE — spawned but not yet settled — off the scope's in-memory live set - // (O(live), synchronous). The terminal statuses are done/failed/cancelled; everything else - // (pending/acquiring/running) is still in flight. This is the concurrency fence's input. + // A supervised tree exposes one shared capacity reading; a caller-owned legacy scope falls back + // to this toolbox's direct-child count. The shared reading is what prevents each nested manager + // from multiplying the same cap independently. const maxLiveWorkers = opts.maxLiveWorkers const isLive = (status: string): boolean => status !== 'done' && status !== 'failed' && status !== 'cancelled' - const liveWorkerCount = (): number => opts.scope.view.nodes.filter((n) => isLive(n.status)).length + const localLiveWorkerCount = (): number => + opts.scope.view.nodes.filter((n) => isLive(n.status)).length + const sharedWorkerCapacity = (): Scope['workerCapacity'] | undefined => { + const scope = opts.scope as Partial> + return scope.workerCapacity + } + const usesTreeWideLimit = (): boolean => { + const capacity = sharedWorkerCapacity() + return capacity !== undefined && capacity.freeSlots !== null + } + const liveWorkerCount = (): number => + usesTreeWideLimit() + ? (sharedWorkerCapacity()?.live ?? localLiveWorkerCount()) + : localLiveWorkerCount() // A snapshot of every still-in-flight worker — the liveness signal a bounded `await_event` // returns when its wait elapses, so the supervisor can tell "worker still running, keep waiting" // apart from "nothing is happening" (the distinction it lost when the unbounded await erred out). - const liveSnapshot = (): Array<{ id: string; status: string; spent: unknown }> => - opts.scope.view.nodes - .filter((n) => isLive(n.status)) - .map((n) => ({ id: n.id, status: n.status, spent: n.spent })) + const projectNodeEvidence = ( + node: Scope['view']['nodes'][number], + resumed = false, + ): Record => ({ + id: node.id, + status: node.status, + ...(node.assignmentId === undefined ? {} : { assignmentId: node.assignmentId }), + ...(node.identity === undefined ? {} : { identity: node.identity }), + ...(node.materialization === undefined ? {} : { materialization: node.materialization }), + ...(node.executionBindings === undefined ? {} : { executionBindings: node.executionBindings }), + spent: node.spent, + ...(node.settledAt === undefined ? {} : { settledAt: node.settledAt }), + ...(node.outRef === undefined ? {} : { outRef: node.outRef }), + ...(resumed ? { resumed: true } : {}), + }) + + const liveSnapshot = (): Array> => + opts.scope.view.nodes.filter((n) => isLive(n.status)).map((n) => projectNodeEvidence(n)) // How many workers the driver could open RIGHT NOW without hitting the simultaneity fence, or // `null` when no cap is set (the conserved pool is then the only fence, so there is no finite @@ -521,7 +871,10 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin // so filling N slots meant emitting N blind tool calls with no feedback telling it to — the // mechanical reason a 5-worker run peaked at 2 live workers. Policy (whether to fill) stays with // the driver; this is only the reading. - const freeWorkerSlots = (): number | null => freeSlots(liveWorkerCount(), maxLiveWorkers) + const freeWorkerSlots = (): number | null => + usesTreeWideLimit() + ? (sharedWorkerCapacity()?.freeSlots ?? null) + : freeSlots(localLiveWorkerCount(), maxLiveWorkers) // The LIVE read of one worker. Guarded because `createCoordinationTools` is bound to a `Scope` // it did not construct — an older or hand-rolled scope may not implement `progress` at all, and @@ -664,10 +1017,10 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin 'Optional per-spawn budget that merges over the per-worker default (per field). ' + 'Only set the ceilings this sub-task needs raised; the conserved pool still fences.', properties: { - maxIterations: { type: 'number' }, - maxTokens: { type: 'number' }, - maxUsd: { type: 'number' }, - deadlineMs: { type: 'number' }, + maxIterations: { type: 'number', minimum: 0 }, + maxTokens: { type: 'number', minimum: 0 }, + maxUsd: { type: 'number', minimum: 0 }, + deadlineMs: { type: 'number', minimum: 0 }, }, }, }, @@ -684,6 +1037,7 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin // touches the pool. The conserved pool bounds TOTAL work; this bounds SIMULTANEOUS work. if ( !keyCompleted && + !usesTreeWideLimit() && maxLiveWorkers !== undefined && maxLiveWorkers > 0 && liveWorkerCount() >= maxLiveWorkers @@ -693,12 +1047,36 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin live: liveWorkerCount(), freeSlots: freeWorkerSlots(), }) - const agent = opts.makeWorkerAgent(a.profile) - const budget = - a.budget === undefined ? opts.perWorker : mergeBudget(opts.perWorker, a.budget) - const res = opts.scope.spawn(agent, a.task, { + const parsedProfile = agentProfileSchema.safeParse(a.profile) + if (!parsedProfile.success) { + return Promise.resolve({ + error: 'invalid-profile' as const, + issues: parsedProfile.error.issues.map((issue) => ({ + path: issue.path.join('.'), + message: issue.message, + })), + }) + } + const profile = deepFreezeDetached(parsedProfile.data) + const task = deepFreezeDetached(a.task) + const label = typeof a.label === 'string' ? a.label : 'worker' + const budget = Object.freeze( + a.budget === undefined ? opts.perWorker : mergeBudget(opts.perWorker, a.budget), + ) + const assignmentId = + key !== undefined ? `key:${key}` : `ordinal:${unkeyedAssignmentOrdinal++}` + const context: WorkerSpawnContext = Object.freeze({ + assignmentId, + parentNodeId: opts.scope.view.root, budget, - label: typeof a.label === 'string' ? a.label : 'worker', + task, + label, + ...(key !== undefined ? { key } : {}), + }) + const res = opts.scope.spawn(() => opts.makeWorkerAgent(profile, context), task, { + budget, + label, + assignmentId, ...(key !== undefined ? { key } : {}), }) // A keyed spawn that resolved to committed work: NOTHING ran — return the finished result @@ -707,13 +1085,12 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin if (res.ok && res.prior?.state === 'completed') { const s = res.prior.settled if (key !== undefined) completedKeys.add(key) + const { id, status, resumed: _resumed, ...evidence } = projectSettled(s) return Promise.resolve({ - workerId: s.handle.id, + workerId: id, resumed: 'completed' as const, - status: 'done' as const, - score: s.verdict?.score ?? 0, - valid: s.verdict?.valid ?? false, - outRef: s.outRef, + status, + ...evidence, live: liveWorkerCount(), freeSlots: freeWorkerSlots(), }) @@ -742,6 +1119,14 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin res.ok ? { workerId: res.handle.id, + assignmentId: res.handle.assignmentId ?? assignmentId, + ...(res.handle.identity === undefined ? {} : { identity: res.handle.identity }), + ...(res.handle.materialization === undefined + ? {} + : { materialization: res.handle.materialization }), + ...(res.handle.executionBindings === undefined + ? {} + : { executionBindings: res.handle.executionBindings }), live: liveWorkerCount(), freeSlots: freeWorkerSlots(), ...priorHistory, @@ -771,19 +1156,16 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin if (!resumed) return { error: `unknown workerId ${JSON.stringify(id)}` } const output = resumed.outRef ? await opts.blobs.get(resumed.outRef) : undefined return { - status: resumed.status, - spent: resumed.spent, + ...projectNodeEvidence(resumed, true), outRef: resumed.outRef ?? null, output: output ?? null, progress: null, - resumed: true, } } const output = node.outRef ? await opts.blobs.get(node.outRef) : undefined const progress = readProgress(id) return { - status: node.status, - spent: node.spent, + ...projectNodeEvidence(node), outRef: node.outRef ?? null, output: output ?? null, progress: progress ?? null, @@ -817,18 +1199,20 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin const workerId = str(a.workerId, 'workerId') const instruction = str(a.instruction, 'instruction') const interrupt = a.interrupt === true - const delivered = opts.scope.send(workerId, { steer: instruction, interrupt }) - await sendDown('steer', { toWorker: workerId, instruction, delivered }) - if (delivered) return { delivered, progress: readProgress(workerId) ?? null } - // Say WHY nothing landed. A silent `delivered:false` is how steering became a no-op: - // the driver could not tell "already finished" from "this runtime has no inbox at all". - const progress = readProgress(workerId) - const reason = !progress - ? 'unknown-worker' - : !progress.live - ? 'already-settled' - : 'runtime-has-no-inbox' - return { delivered, reason, progress: progress ?? null } + const authorized = authorizeInstruction('steer', workerId, instruction, interrupt) + await recordInstruction(authorized) + const delivery = await attemptDelivery(authorized, { + steer: authorized.instruction, + interrupt, + }) + if (delivery.delivered) { + return { delivered: true, progress: readProgress(workerId) ?? null } + } + return { + delivered: false, + reason: delivery.outcome, + progress: readProgress(workerId) ?? null, + } }, }, { @@ -908,24 +1292,45 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin const questionId = str(a.questionId, 'questionId') if (typeof a.answer === 'string' && a.answer.length > 0) { const answer = a.answer - const question = decideQuestion(questionId, { - kind: 'answer', - answer, - by: typeof a.by === 'string' && a.by.length > 0 ? a.by : 'user', - }) + const pendingQuestion = questions.find((question) => question.id === questionId) + if (pendingQuestion === undefined) { + throw new Error(`unknown questionId ${JSON.stringify(questionId)}`) + } // Route the answer DOWN to the worker that asked, unparking it, and record the down-leg. // A blocking question parked the worker, so deliver forcefully — it should resume on the // answer immediately, not wait for its next step boundary. - const interrupt = question.urgency === 'blocks-run' || question.urgency === 'blocks-step' - const delivered = opts.scope.send(question.from, { answer, questionId, interrupt }) - await sendDown( + const interrupt = + pendingQuestion.urgency === 'blocks-run' || pendingQuestion.urgency === 'blocks-step' + const authorized = authorizeInstruction( 'answer', - { toWorker: question.from, instruction: answer, delivered }, + pendingQuestion.from, + answer, + interrupt, questionId, ) + await recordInstruction(authorized) + const delivery = await attemptDelivery(authorized, { + answer: authorized.instruction, + questionId, + interrupt, + }) + // Authorization is evidence of allowed bytes, not proof the blocked worker received them. + // Resolve the question only after the durable delivery outcome says the live inbox accepted + // the answer. A refusal stays open both in this process and after replay. + const question = delivery.delivered + ? decideQuestion(questionId, { + kind: 'answer', + answer: authorized.instruction, + by: typeof a.by === 'string' && a.by.length > 0 ? a.by : 'user', + }) + : pendingQuestion // Surface `delivered` like steer_agent — the caller must see whether the answer actually // reached a live worker (false when it already settled or has no inbox). - return { question, delivered } + return { + question, + delivered: delivery.delivered, + ...(delivery.delivered ? {} : { reason: delivery.outcome }), + } } if (typeof a.deferReason === 'string' && a.deferReason.length > 0) { return Promise.resolve({ @@ -1041,6 +1446,7 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin return { tools, + ready, history: () => bus.history(), raiseFinding: (finding) => bus.publish({ type: 'finding', finding }).then(() => undefined), stats: () => bus.stats(), @@ -1051,3 +1457,14 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin drainResolved, } } + +function deepFreezeDetached(value: T): T { + return deepFreeze(structuredClone(value)) +} + +function deepFreeze(value: T, seen = new Set()): T { + if (value === null || typeof value !== 'object' || seen.has(value)) return value + seen.add(value) + for (const child of Object.values(value as Record)) deepFreeze(child, seen) + return Object.freeze(value) +} diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 519c8c55..bcf3f4a5 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -46,17 +46,24 @@ export { pendingWaits, replaySpawnTree, } from '../durable/spawn-journal' -// The typed coordination-bus event (up: settled/question/finding; down: steer/answer) — surfaced -// here so a host folding the bus onto its own timeline (the supervise-topology observability) can +// The typed coordination-bus event (up: settled/question/finding; authorized instruction receipt; +// down: steer/answer delivery outcome) — surfaced here so a host folding the bus onto its own timeline can // type its `onEvent` subscriber without reaching into the `/mcp` subpath. `MakeWorkerAgent` rides // alongside it: the worker-seam type `supervise`/`workerFromBackend` traffic in, so a host authoring // its own seam types it from the loop layer rather than the `/mcp` subpath. export type { AnalystFindingEvent, AnalystRegistry, + AuthorizeDownMessage, + AuthorizedDownMessage, + ContinuationInstruction, CoordinationEvent, + DownMessageAuthorizationInput, + DownMessageDeliveryAttempt, + DownMessageDeliveryOutcome, DownMessageEvent, MakeWorkerAgent, + WorkerSpawnContext, } from './../mcp/tools/coordination' export { DEFAULT_AWAIT_EVENT_TIMEOUT_MS } from './../mcp/tools/coordination' export type { WorktreeCheckRunner, WorktreeHarnessResult } from './../mcp/worktree-harness' @@ -506,6 +513,7 @@ export { } from './supervise/authoring' export { type BudgetPool, + type BudgetPoolRestore, type BudgetReadout, createBudgetPool, type ReservationTicket, @@ -515,18 +523,21 @@ export { // settlement `valid` reflects a deployable deliverable check (a test/judge), never self-report. export { type DeliverableSpec, gateOnDeliverable } from './supervise/completion-gate' // The CHEAP / offline driver: an in-process router-tools loop that drives the coordination -// verbs over the Scope (no box, no creds). The CAPABLE driver is a sandbox agent with the -// coordination verbs mounted as an MCP — this is the low-cost + offline-testable variant. +// verbs over the Scope (no box, no creds). The CAPABLE driver is an external harness with the +// coordination verbs mounted as an MCP: `supervise()` wires a local bridge automatically, while a +// remote sandbox requires an explicit reachable `driveHarness`. export { type DriverAgentOptions, driverAgent, finalizeBestDelivered, } from './supervise/coordination-driver' -// The durable coordination side-log a file-backed `RunContext` carries: the questions and analyst -// findings the spawn journal does not record, replayed into a resumed driver so a restarted -// coordinator keeps the coordination context its workers produced. +// The durable coordination side-log a file-backed `RunContext` carries: questions, findings, answer +// decisions, and authorized continuation receipts the spawn journal does not own. Receipts persist +// as evidence and are never auto-delivered to a replacement worker. export { + type CoordinationDeliveryEvidence, type CoordinationLog, + type CoordinationOwnerId, FileCoordinationLog, type PriorCoordination, } from './supervise/coordination-log' @@ -592,7 +603,7 @@ export { // drains it at the step boundary + before settle (queued) or aborts the turn (forceful interrupt). export { createInbox, type Inbox, type InboxMessage } from './supervise/inbox' // The fail-loud model-subset guard the front doors call: restrict a run to a chosen set of models. -export { assertModelAllowed } from './supervise/model-policy' +export { assertModelAllowed, assertProfileModelsAllowed } from './supervise/model-policy' // The mechanical patch gate as a generic DeliverableSpec over the worktree-CLI patch artifact: // no-op / always-on secret-path floor / forbidden-path / diff-size + required test/typecheck pass. export { type PatchDeliverableOptions, patchDelivered } from './supervise/patch-deliverable' @@ -683,18 +694,25 @@ export { // sensible defaults (blobs/perWorker/journal/executors). `workerFromBackend` derives the worker seam // from a backend config + an optional completion oracle (settled⟺delivered). export { + type AuthorizedSpawn, + DEFAULT_AUTHORED_PROFILE_SECURITY_POLICY, type SuperviseOptions, supervise, workerFromBackend, } from './supervise/supervise' export { createSupervisor } from './supervise/supervisor' // Build a supervisor FROM its profile: the brain is resolved from `profile.harness` like -// `createExecutor({backend})` resolves a worker — `null` → the in-process router tool-loop, +// `createExecutor({backend})` resolves a worker — omitted/`cli-base` → the in-process router tool-loop, // a coding-CLI harness → a sandboxed harness driving the coordination verbs. No hand-built brain. export { type DriveHarness, + type ObserveSupervisorNodeEvent, + type ResolveSupervisorTools, type SupervisorAgentDeps, + type SupervisorNodeContext, + type SupervisorNodeContextSeed, type SupervisorProfile, + type SupervisorToolDescriptor, supervisorAgent, } from './supervise/supervisor-agent' // The substrate-agnostic trace source: a worker's tool calls as agent-eval `ToolSpan`s, from an @@ -713,22 +731,33 @@ export { export { analyzeTrace, type TrajectoryAnalysis } from './supervise/trajectory-recorder' export type { Agent, + AgentExecutionRef, AgentSpec, Budget, + ExecutionBindingReceipt, Executor, + ExecutorAccounting, ExecutorContext, + ExecutorExecutionBinding, ExecutorFactory, + ExecutorMaterialization, + ExecutorNodeContext, ExecutorRegistry, ExecutorResult, Handle, + MaterializedExecutionIdentity, + MaterializedModelIdentity, + NodeExecutionIdentity, NodeId, NodeSnapshot, NodeStatus, + ProfileMaterializationReceipt, Restart, ResultBlobStore, ResumedKeyState, ResumedWork, RootHandle, + RootMaterialization, RootSignal, Runtime, Scope, @@ -743,6 +772,7 @@ export type { Supervisor, SupervisorOpts, TreeView, + UnknownMaterializationReason, UsageEvent, WaitOpts, WidenGate, diff --git a/src/runtime/supervise/budget.ts b/src/runtime/supervise/budget.ts index b3c88eba..4b56e00c 100644 --- a/src/runtime/supervise/budget.ts +++ b/src/runtime/supervise/budget.ts @@ -39,16 +39,70 @@ export interface ReservationTicket { /** Post-reservation pool readout — the shape `Scope.budget` exposes. `tokensLeft`, * `usdLeft`, and `reservedTokens` reflect committed-but-unsettled reservations; * `deadlineMs` is the ABSOLUTE wall-clock deadline (0 when the root set none). + * `iterationsLeft` is the remaining iteration capacity. `tokensKnown`/`usdKnown` are false when + * any prior or current execution lacks measured telemetry; the numeric remainder may still be a + * safe admission bound when an in-doubt child has already been charged at its full reservation. * `usdCapped` distinguishes a real `usdLeft <= 0` exhaustion from an uncapped pool (which always * reads `usdLeft: 0`) — the in-loop guard needs it to bound a usd-capped driver. */ export type BudgetReadout = Readonly<{ tokensLeft: number + tokensKnown: boolean usdLeft: number usdCapped: boolean + usdKnown: boolean + iterationsLeft: number deadlineMs: number reservedTokens: number }> +/** State recovered from a prior process before new work is admitted. `committed` is measured spend + * already present in the durable journal. Each `uncertainReservation` is a child that was recorded + * as started but never recorded as settled: its full declared ceiling is charged conservatively, + * while the public readout remains explicitly unknown. */ +export interface BudgetPoolRestore { + readonly committed?: Spend + readonly uncertainReservations?: ReadonlyArray + /** Original absolute deadline from the first process. It may never slide on restart. */ + readonly absoluteDeadlineMs?: number +} + +/** Reject malformed ceilings before they can mint capacity through negative reservations. */ +export function assertValidBudget(budget: Budget, label = 'budget'): void { + const safeInteger = (value: number, field: string) => { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`${label}.${field} must be a non-negative safe integer`) + } + } + const finiteNonNegative = (value: number, field: string) => { + if (!Number.isFinite(value) || value < 0) { + throw new Error(`${label}.${field} must be a non-negative finite number`) + } + } + safeInteger(budget.maxIterations, 'maxIterations') + safeInteger(budget.maxTokens, 'maxTokens') + if (budget.maxUsd !== undefined) finiteNonNegative(budget.maxUsd, 'maxUsd') + if (budget.deadlineMs !== undefined) finiteNonNegative(budget.deadlineMs, 'deadlineMs') +} + +function assertValidSpend(spend: Spend, label: string): void { + if (!Number.isSafeInteger(spend.iterations) || spend.iterations < 0) { + throw new Error(`${label}.iterations must be a non-negative safe integer`) + } + for (const [field, value] of Object.entries(spend.tokens)) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new Error(`${label}.tokens.${field} must be a non-negative safe integer`) + } + } + for (const [field, value] of [ + ['usd', spend.usd], + ['ms', spend.ms], + ] as const) { + if (!Number.isFinite(value) || value < 0) { + throw new Error(`${label}.${field} must be a non-negative finite number`) + } + } +} + export interface BudgetPool { /** * Atomically reserve a child's full ceiling from the free balance. Fails closed @@ -130,25 +184,47 @@ function totalTokens(usage: LoopTokenUsage): number { /** * Create a conserved reservation pool from a root `Budget`. `now()` is injected so the * deadline readout is deterministic; defaults to `Date.now` for non-test callers. The - * absolute deadline is fixed at construction (`now() + budget.deadlineMs`) so the - * readout's `deadlineMs` is a stable wall-clock instant, not a shrinking remainder. + * absolute deadline for a fresh pool is fixed at construction (`now() + budget.deadlineMs`). A + * restored pool instead retains `restore.absoluteDeadlineMs`, so restart never slides the original + * wall-clock limit. The readout is an absolute instant, not a shrinking remainder. */ -export function createBudgetPool(root: Budget, now: () => number = Date.now): BudgetPool { +export function createBudgetPool( + root: Budget, + now: () => number = Date.now, + restore: BudgetPoolRestore = {}, +): BudgetPool { + assertValidBudget(root, 'root budget') + if ( + restore.absoluteDeadlineMs !== undefined && + (!Number.isFinite(restore.absoluteDeadlineMs) || restore.absoluteDeadlineMs < 0) + ) { + throw new Error('budget restore.absoluteDeadlineMs must be a non-negative finite number') + } // free + reserved + committed ≡ root totals, per channel, always. let freeTokens = root.maxTokens let reservedTokens = 0 let committedTokens = 0 + let tokensKnown = true const usdCapped = root.maxUsd !== undefined let freeUsd = root.maxUsd ?? 0 let reservedUsd = 0 let committedUsd = 0 + let usdKnown = true + + // `Known` describes the final telemetry. `AdmissionSafe` describes whether the remaining + // numeric capacity is still a sound upper bound. A killed child makes telemetry unknown but its + // declared reservation can still be charged in full, leaving the rest safe to use. An executor + // that reports wholly unbounded usage consumes the remaining channel and makes it unsafe. + let tokensAdmissionSafe = true + let usdAdmissionSafe = true let freeIterations = root.maxIterations let reservedIterations = 0 let committedIterations = 0 - const absoluteDeadlineMs = root.deadlineMs !== undefined ? now() + root.deadlineMs : 0 + const absoluteDeadlineMs = + restore.absoluteDeadlineMs ?? (root.deadlineMs !== undefined ? now() + root.deadlineMs : 0) let nextTicketId = 0 const open = new Set() @@ -156,6 +232,10 @@ export function createBudgetPool(root: Budget, now: () => number = Date.now): Bu function reserve( b: Budget, ): { ok: true; ticket: ReservationTicket } | { ok: false; reason: 'budget-exhausted' } { + assertValidBudget(b, 'reservation budget') + if (!tokensAdmissionSafe || (usdCapped && !usdAdmissionSafe)) { + return { ok: false, reason: 'budget-exhausted' } + } const wantTokens = b.maxTokens const wantUsd = b.maxUsd ?? 0 const wantIterations = b.maxIterations @@ -188,47 +268,45 @@ export function createBudgetPool(root: Budget, now: () => number = Date.now): Bu if (!open.has(ticket.id)) { throw new Error(`budget pool: reconcile of unknown or already-settled ticket ${ticket.id}`) } - open.delete(ticket.id) + assertValidSpend(spent, `budget pool ticket ${ticket.id} spend`) const { tokens: rTokens, usd: rUsd, iterations: rIterations } = ticket.reserved - if (usdCapped && spent.usdKnown === false) { - throw new Error( - `budget pool: ticket ${ticket.id} reported unknown dollar cost under a dollar-capped budget`, - ) - } + const unknownTokens = spent.tokensKnown === false + const unknownUsd = spent.usdKnown === false + const unknownCappedUsd = usdCapped && unknownUsd + if (unknownUsd) usdKnown = false - // Clamp actual spend to the reservation: a child must never commit more than it - // reserved (that would overdraw the conserved pool). Over-spend is a fail-loud bug. const spentTokens = totalTokens(spent.tokens) - if (spentTokens > rTokens) { - throw new Error( - `budget pool: ticket ${ticket.id} spent ${spentTokens} tokens > reserved ${rTokens}`, - ) - } - if (spent.iterations > rIterations) { - throw new Error( - `budget pool: ticket ${ticket.id} spent ${spent.iterations} iterations > reserved ${rIterations}`, - ) - } - // USD is conserved ONLY when the root declared a ceiling. `maxUsd` is optional: when no - // root ceiling exists, usd is an OBSERVED quantity (committed for accounting), never a - // budgeted constraint — so an unset ceiling must not behave as a hard $0 limit that - // fail-closes a real priced spend. The over-spend clamp applies only to a capped pool. - if (usdCapped && spent.usd > rUsd) { - throw new Error(`budget pool: ticket ${ticket.id} spent $${spent.usd} > reserved $${rUsd}`) - } - - // Release the whole reservation, then commit actual spend; the difference is the - // refund that flows back to `free`. + // Release the reservation and record ACTUAL spend. External providers can report an + // overshoot after it has happened; free capacity goes negative so later admission stops. + // Telemetry is never clamped to make the bookkeeping look within budget. + open.delete(ticket.id) reservedTokens -= rTokens - committedTokens += spentTokens - freeTokens += rTokens - spentTokens + if (unknownTokens) { + tokensKnown = false + tokensAdmissionSafe = false + committedTokens += rTokens + freeTokens + freeTokens = 0 + } else { + committedTokens += spentTokens + freeTokens += rTokens - spentTokens + } reservedIterations -= rIterations committedIterations += spent.iterations freeIterations += rIterations - spent.iterations - if (usdCapped && rUsd > 0) { + if (unknownCappedUsd) { + // The work already ran, but its dollar cost cannot be compared with the declared ceiling. + // Close the ticket and consume all currently-free dollar capacity so no later work can treat + // the unknown amount as $0. The durable Spend keeps `usdKnown:false`; this is only the live + // pool's fail-closed upper bound. + reservedUsd -= rUsd + usdKnown = false + usdAdmissionSafe = false + committedUsd += rUsd + freeUsd + freeUsd = 0 + } else if (usdCapped) { reservedUsd -= rUsd committedUsd += spent.usd freeUsd += rUsd - spent.usd @@ -237,14 +315,20 @@ export function createBudgetPool(root: Budget, now: () => number = Date.now): Bu // without touching the reservation channel — usd is accounted, not conserved here. committedUsd += spent.usd } - } - - function observe(spend: Spend): void { - if (usdCapped && spend.usdKnown === false) { + if (unknownTokens || unknownCappedUsd) { throw new Error( - 'budget pool: cannot observe unknown dollar cost under a dollar-capped budget', + `budget pool: ticket ${ticket.id} reported unknown ${[ + ...(unknownTokens ? ['token usage'] : []), + ...(unknownCappedUsd ? ['dollar cost'] : []), + ].join(' and ')} under a capped budget`, ) } + } + + function observe(spend: Spend): void { + assertValidSpend(spend, 'observed spend') + const unknownTokens = spend.tokensKnown === false + const unknownCappedUsd = usdCapped && spend.usdKnown === false const tokens = totalTokens(spend.tokens) // Direct free → committed debit (no reservation ticket). `free` may go negative on overspend — // that is honest; the readout then reports exhaustion and the in-loop guard halts the driver. @@ -252,17 +336,51 @@ export function createBudgetPool(root: Budget, now: () => number = Date.now): Bu // `Scope.meter`); this debit exists only to make the live `readout()` reflect driver inference. freeTokens -= tokens committedTokens += tokens + if (unknownTokens) { + tokensKnown = false + tokensAdmissionSafe = false + if (freeTokens > 0) { + committedTokens += freeTokens + freeTokens = 0 + } + } freeIterations -= spend.iterations committedIterations += spend.iterations - committedUsd += spend.usd - if (usdCapped) freeUsd -= spend.usd + if (unknownCappedUsd) { + usdKnown = false + usdAdmissionSafe = false + committedUsd += spend.usd + freeUsd -= spend.usd + if (freeUsd > 0) { + committedUsd += freeUsd + freeUsd = 0 + } + } else { + if (spend.usdKnown === false) usdKnown = false + committedUsd += spend.usd + if (usdCapped) freeUsd -= spend.usd + } + if (unknownTokens || unknownCappedUsd) { + throw new Error( + `budget pool: cannot observe unknown ${[ + ...(unknownTokens ? ['token usage'] : []), + ...(unknownCappedUsd ? ['dollar cost'] : []), + ].join(' and ')} under a capped budget`, + ) + } } function readout(): BudgetReadout { return { + // This is safe remaining capacity, not a claim that prior telemetry was measured. A resumed + // in-doubt child has already been charged at its full ceiling, so unrelated capacity can + // remain usable while `tokensKnown:false` keeps the report honest. tokensLeft: freeTokens, + tokensKnown, usdLeft: usdCapped ? freeUsd : 0, usdCapped, + usdKnown, + iterationsLeft: freeIterations, deadlineMs: absoluteDeadlineMs, reservedTokens, } @@ -276,6 +394,42 @@ export function createBudgetPool(root: Budget, now: () => number = Date.now): Bu } } + // Reconstruct the conserved balances before exposing the pool. Measured prior spend is debited + // exactly once. A started-without-terminal child is charged at its entire reservation: this + // never mints capacity on restart, but unlike treating unknown as zero it can leave unrelated + // capacity usable. The false known flags remain visible in every later report. + if (restore.committed !== undefined) { + // `observe` deliberately throws only after consuming unknown capped telemetry, but it also + // rejects malformed spend before mutation. Validate first so the narrow catch below cannot + // turn a corrupt durable record into a zero-cost restart. + assertValidSpend(restore.committed, 'budget restore committed') + try { + observe(restore.committed) + } catch { + // `observe` has already consumed the unsafe channel before it reports the unknown telemetry. + } + } + for (const [index, uncertain] of (restore.uncertainReservations ?? []).entries()) { + assertValidBudget(uncertain, `budget restore uncertainReservations[${index}]`) + tokensKnown = false + usdKnown = false + freeTokens -= uncertain.maxTokens + committedTokens += uncertain.maxTokens + freeIterations -= uncertain.maxIterations + committedIterations += uncertain.maxIterations + if (usdCapped) { + if (uncertain.maxUsd === undefined) { + // No child dollar ceiling existed, so the root's remaining dollar capacity is not safe. + committedUsd += freeUsd + freeUsd = 0 + usdAdmissionSafe = false + } else { + freeUsd -= uncertain.maxUsd + committedUsd += uncertain.maxUsd + } + } + } + return { reserve, reconcile, diff --git a/src/runtime/supervise/coordination-driver.ts b/src/runtime/supervise/coordination-driver.ts index ef391869..e002026d 100644 --- a/src/runtime/supervise/coordination-driver.ts +++ b/src/runtime/supervise/coordination-driver.ts @@ -30,6 +30,7 @@ import { RuntimeRunStateError, ValidationError } from '../../errors' import type { McpToolDescriptor } from '../../mcp/server' import { type AnalystRegistry, + type AuthorizeDownMessage, type CoordinationEvent, coordinationVerbNames, createCoordinationTools, @@ -45,6 +46,7 @@ import { type ToolLoopCompactionOptions, } from '../tool-loop' import type { PriorCoordination } from './coordination-log' +import type { BusRecord } from './event-bus' import { bestDelivered, pickBestDelivered, @@ -79,6 +81,7 @@ export interface DriverAgentOptions { readonly blobs: ResultBlobStore /** Resolve a spawned `profile` to a worker LEAF or a driver child (the recursion seam). */ readonly makeWorkerAgent: MakeWorkerAgent + readonly authorizeDownMessage?: AuthorizeDownMessage /** Per-child budget reserved from the conserved pool on each spawn. */ readonly perWorker: Budget /** Hard cap on simultaneously-LIVE workers — `spawn_agent` fails closed once this many are in @@ -101,6 +104,9 @@ export interface DriverAgentOptions { /** The driver's stance — a string, or built from the task (the worker-driver prompt / * the generator). INJECTED so the prompt is a pluggable, optimizable role. */ readonly systemPrompt: string | ((task: unknown) => string) + /** Product-selected tools already bound to this exact supervisor node. The same descriptors are + * served over MCP for external supervisors; this arm projects them into router ToolSpecs. */ + readonly nodeTools?: ReadonlyArray /** WORK tools the driver may call DIRECTLY (alongside the coordination verbs) — so the driver is * not a pure manager but a full agent that can ACT (do simple work itself) OR SPAWN (delegate). * Each is a router tool spec; their names must not collide with the coordination verbs. Pair with @@ -151,12 +157,18 @@ export interface DriverAgentOptions { * off (no behavior change). `distill` defaults to a self-summary authored by the brain combined * with the factual settled-worker roster; override to supply your own. */ readonly compaction?: ToolLoopCompactionOptions - /** Pass-through subscriber for every coordination bus event (settled / question / finding / - * steer / answer) — what a durable caller hooks its coordination log onto. Omit = no observer. */ - readonly onEvent?: (event: CoordinationEvent) => void | Promise - /** Questions + findings a durable coordination log replayed from a prior process of this run. - * Questions seed the ledger (`list_questions`, blocking-stop policy); both feed the resume - * brief. Omit = fresh (every run that is not a resume). */ + /** Pass-through subscriber for every coordination bus event: settled/question/finding, + * pre-delivery instruction receipts, and steer/answer delivery outcomes. A durable caller uses + * this to append the coordination log. Omit = no observer. */ + readonly onEvent?: ( + event: CoordinationEvent, + record: BusRecord, + ) => void | Promise + /** Re-publish resume-time settlements through the awaited observer before the first brain turn. */ + readonly replaySettlements?: boolean + /** Questions, findings, and authorized continuation receipts loaded from a prior process. + * Questions seed the ledger (`list_questions`, blocking-stop policy); all three feed the resume + * brief. Continuation receipts are evidence only and are never auto-delivered. Omit = fresh. */ readonly priorCoordination?: PriorCoordination /** How the settled-worker ledger becomes the run's output. Default `bestDelivered` — the single * highest-scoring DELIVERED child (the exact keep-best every existing caller had). Runs under @@ -209,10 +221,13 @@ const runawayTripwireTurns = 2000 * overspend usd up to the turn tripwire). */ function poolStarved(scope: Scope, perWorker: Budget): boolean { const b = scope.budget - if (b.reservedTokens > 0) return false // a child is in flight — await it, don't finalize early + if (scope.view.inFlight > 0 || scope.view.waiting > 0) return false const tokenStarved = b.tokensLeft < perWorker.maxTokens - const usdStarved = b.usdCapped && b.usdLeft <= 0 - return tokenStarved || usdStarved + const iterationStarved = b.iterationsLeft <= 0 + const usdStarved = + b.usdCapped && + (b.usdLeft <= 0 || (perWorker.maxUsd !== undefined && b.usdLeft < perWorker.maxUsd)) + return tokenStarved || iterationStarved || usdStarved } /** The absolute wall-clock deadline (when the root set one) has passed. */ @@ -278,12 +293,21 @@ export function driverAgent(opts: DriverAgentOptions): Agent { // Validate against the reserved verb set HERE (construction), so the conflict fails loud — not // buried inside act() where the supervisor would swallow the throw into a quiet no-winner. const reserved = new Set(coordinationVerbNames) + for (const tool of opts.nodeTools ?? []) { + if (reserved.has(tool.name)) { + throw new ValidationError( + `driverAgent: node tool "${tool.name}" collides with a coordination verb or another node tool`, + ) + } + reserved.add(tool.name) + } for (const t of opts.extraTools ?? []) { if (reserved.has(t.name)) { throw new ValidationError( - `driverAgent: extra work tool "${t.name}" collides with a coordination verb`, + `driverAgent: extra work tool "${t.name}" collides with a coordination verb or node tool`, ) } + reserved.add(t.name) } // Fail loud on a nonsensical cap: a negative maxTurns would silently run zero turns and // finalize an empty no-winner — a silent zero the house rules forbid. @@ -306,6 +330,7 @@ export function driverAgent(opts: DriverAgentOptions): Agent { scope, blobs: opts.blobs, makeWorkerAgent: opts.makeWorkerAgent, + ...(opts.authorizeDownMessage ? { authorizeDownMessage: opts.authorizeDownMessage } : {}), perWorker: opts.perWorker, ...(opts.maxLiveWorkers !== undefined ? { maxLiveWorkers: opts.maxLiveWorkers } : {}), ...(opts.analysts ? { analysts: opts.analysts } : {}), @@ -313,10 +338,12 @@ export function driverAgent(opts: DriverAgentOptions): Agent { ...(opts.watchWorkers ? { watchWorkers: opts.watchWorkers } : {}), ...(opts.stallAfterMs !== undefined ? { stallAfterMs: opts.stallAfterMs } : {}), ...(opts.onEvent ? { onEvent: opts.onEvent } : {}), + ...(opts.replaySettlements ? { replaySettlements: true } : {}), ...(opts.priorCoordination?.questions.length ? { priorQuestions: opts.priorCoordination.questions } : {}), }) + await coord.ready() // Resume-first: re-establish the prior process's supervision state BEFORE the first brain // turn — its armed-but-never-woken waits become live again on their ORIGINAL deadlines // (they settle through the same cursor `await_event` drains). Fail loud on a wait that @@ -329,12 +356,18 @@ export function driverAgent(opts: DriverAgentOptions): Agent { ) } } - const byName = new Map(coord.tools.map((t) => [t.name, t])) + const byName = new Map( + [...coord.tools, ...(opts.nodeTools ?? [])].map((t) => [t.name, t]), + ) const toolSpecs: ToolSpec[] = [ ...coord.tools.map((t) => ({ type: 'function' as const, function: { name: t.name, description: t.description, parameters: t.inputSchema }, })), + ...(opts.nodeTools ?? []).map((t) => ({ + type: 'function' as const, + function: { name: t.name, description: t.description, parameters: t.inputSchema }, + })), // Work tools the driver calls DIRECTLY — so it can ACT, not only delegate. ...(opts.extraTools ?? []).map((t) => ({ type: 'function' as const, @@ -442,7 +475,14 @@ export function driverAgent(opts: DriverAgentOptions): Agent { // work instead of re-planning (and re-paying) from scratch. ...(scope.resume ? [{ role: 'user', content: resumeBrief(scope.resume, opts.priorCoordination) }] - : []), + : hasPriorCoordination(opts.priorCoordination) + ? [ + { + role: 'user', + content: priorCoordinationBrief(opts.priorCoordination as PriorCoordination), + }, + ] + : []), ], maxTurns, // The conserved-pool + deadline + external-stop bound (what maxTurns=0 relies on): a driver @@ -500,8 +540,9 @@ export function driverAgent(opts: DriverAgentOptions): Agent { /** * The factual context a resumed driver starts from — everything the durable stores prove about * the prior process(es): committed settlements, per-key states (completed / lost / failed), - * re-armed waits, carried-over questions and findings, and the spend already paid. Injected as - * the brain's first user-context on a resumed run so it continues from the unresolved work. + * re-armed waits, carried-over questions/findings/continuation receipts, and spend already paid. + * Injected as the brain's first user-context on a resumed run so it continues from unresolved work; + * old continuation receipts are evidence and are never auto-delivered. */ function resumeBrief(resume: ResumedWork, prior?: PriorCoordination): string { const lines: string[] = [ @@ -554,6 +595,38 @@ function resumeBrief(resume: ResumedWork, prior?: PriorCoordination): s ...resume.waits.map((w) => `- ${w.label} (${w.spec.kind})`), ) } + appendPriorCoordination(lines, prior) + const spent = resume.priorSpend + lines.push( + '', + 'Budget the run ALREADY spent before this process (it counts toward the run total):', + `- child work: tokens in=${spent.childWork.tokens.input} out=${spent.childWork.tokens.output}, usd=${spent.childWork.usd}, iterations=${spent.childWork.iterations}`, + `- driver inference: tokens in=${spent.driverInference.tokens.input} out=${spent.driverInference.tokens.output}, usd=${spent.driverInference.usd}`, + ) + return lines.join('\n') +} + +function hasPriorCoordination(prior?: PriorCoordination): boolean { + return ( + prior !== undefined && + (prior.questions.length > 0 || + prior.findings.length > 0 || + prior.continuations.length > 0 || + prior.deliveryEvidence.length > 0) + ) +} + +function priorCoordinationBrief(prior: PriorCoordination): string { + const lines = [ + 'PRIOR COORDINATION EVIDENCE: this logical supervisor ran in an earlier process.', + 'Use the evidence below as context. Never auto-deliver an old continuation; issue a new', + 'authorized instruction only when current live state still warrants it.', + ] + appendPriorCoordination(lines, prior) + return lines.join('\n') +} + +function appendPriorCoordination(lines: string[], prior?: PriorCoordination): void { const openQuestions = (prior?.questions ?? []).filter( (q) => q.status === 'open' || q.status === 'escalated', ) @@ -575,14 +648,31 @@ function resumeBrief(resume: ResumedWork, prior?: PriorCoordination): s ), ) } - const spent = resume.priorSpend - lines.push( - '', - 'Budget the run ALREADY spent before this process (it counts toward the run total):', - `- child work: tokens in=${spent.childWork.tokens.input} out=${spent.childWork.tokens.output}, usd=${spent.childWork.usd}, iterations=${spent.childWork.iterations}`, - `- driver inference: tokens in=${spent.driverInference.tokens.input} out=${spent.driverInference.tokens.output}, usd=${spent.driverInference.usd}`, - ) - return lines.join('\n') + if ((prior?.continuations.length ?? 0) > 0) { + const attempts = new Set( + (prior?.deliveryEvidence ?? []) + .filter((event) => event.type === 'delivery-attempt') + .map((event) => event.attempt.receiptId), + ) + const outcomes = new Map( + (prior?.deliveryEvidence ?? []) + .filter((event) => event.type === 'steer' || event.type === 'answer') + .map((event) => [event.down.receiptId, event.down.outcome] as const), + ) + lines.push( + '', + 'Authorized continuations committed by the prior process (evidence only; never replayed automatically):', + ...(prior?.continuations ?? []).map((continuation) => { + const outcome = outcomes.get(continuation.receiptId) + const delivery = + outcome ?? + (attempts.has(continuation.receiptId) + ? 'unknown-after-crash' + : 'not-attempted-before-crash') + return `- receipt=${continuation.receiptId}, ${continuation.kind} → ${continuation.toWorker}, instruction=${continuation.instructionDigest}, delivery=${delivery}` + }), + ) + } } /** Run a work tool. A throw is data to the driver (it can recover next turn), not a crash — fold diff --git a/src/runtime/supervise/coordination-mcp.ts b/src/runtime/supervise/coordination-mcp.ts index 12b4a8b0..ccc3bfad 100644 --- a/src/runtime/supervise/coordination-mcp.ts +++ b/src/runtime/supervise/coordination-mcp.ts @@ -20,9 +20,10 @@ */ import { createServer, type Server } from 'node:http' -import { createMcpServer } from '../../mcp/server' +import { createMcpServer, type McpToolDescriptor } from '../../mcp/server' import { type AnalystRegistry, + type AuthorizeDownMessage, type CoordinationEvent, type CoordinationTools, createCoordinationTools, @@ -33,6 +34,7 @@ import { type SettledWorker, type WorkerWatchOptions, } from '../../mcp/tools/coordination' +import type { BusRecord } from './event-bus' import type { Budget, ResultBlobStore, Scope } from './types' export interface CoordinationMcpHandle { @@ -45,7 +47,7 @@ export interface CoordinationMcpHandle { * `settled()` for a finalize, so a delivered child the harness never awaited is not lost. */ drainResolved: CoordinationTools['drainResolved'] isStopped(): boolean - /** The full ordered bus-event log — observability audit + replay trail. */ + /** The full ordered bus-event log for current-process observability and audit evidence. */ history: CoordinationTools['history'] /** Bus throughput counters for live dashboards. */ stats: CoordinationTools['stats'] @@ -60,6 +62,7 @@ export async function serveCoordinationMcp(opts: { scope: Scope blobs: ResultBlobStore makeWorkerAgent: MakeWorkerAgent + authorizeDownMessage?: AuthorizeDownMessage perWorker: Budget /** Hard cap on simultaneously-LIVE workers — `spawn_agent` fails closed once this many are in * flight (a concurrency fence on top of the conserved-pool fence). Omit/`<= 0` = no cap. */ @@ -78,16 +81,23 @@ export async function serveCoordinationMcp(opts: { watchWorkers?: WorkerWatchOptions /** Idle time after which `observe_agent` reports a worker as stalled. */ stallAfterMs?: number - /** Pass-through subscriber for every bus event (settled / question / finding). */ - onEvent?: (event: CoordinationEvent) => void | Promise + /** Pass-through subscriber for every bus event, including pre-delivery instruction receipts and + * steer/answer delivery outcomes. */ + onEvent?: (event: CoordinationEvent, record: BusRecord) => void | Promise + /** Re-publish resume-time settlements through the awaited observer before this server listens. */ + replaySettlements?: boolean questionPolicy?: QuestionPolicy /** Questions replayed from a prior process of this run — seeds the question ledger. */ priorQuestions?: ReadonlyArray + /** Product-selected tools already bound to this exact supervisor node. They share this server + * with the coordination verbs, so the existing MCP duplicate-name guard applies before listen. */ + nodeTools?: ReadonlyArray }): Promise { const coord = createCoordinationTools({ scope: opts.scope, blobs: opts.blobs, makeWorkerAgent: opts.makeWorkerAgent, + ...(opts.authorizeDownMessage ? { authorizeDownMessage: opts.authorizeDownMessage } : {}), perWorker: opts.perWorker, ...(opts.maxLiveWorkers !== undefined ? { maxLiveWorkers: opts.maxLiveWorkers } : {}), awaitTimeoutMs: opts.awaitTimeoutMs ?? DEFAULT_AWAIT_EVENT_TIMEOUT_MS, @@ -96,10 +106,15 @@ export async function serveCoordinationMcp(opts: { ...(opts.watchWorkers ? { watchWorkers: opts.watchWorkers } : {}), ...(opts.stallAfterMs !== undefined ? { stallAfterMs: opts.stallAfterMs } : {}), ...(opts.onEvent ? { onEvent: opts.onEvent } : {}), + ...(opts.replaySettlements ? { replaySettlements: true } : {}), ...(opts.questionPolicy ? { questionPolicy: opts.questionPolicy } : {}), ...(opts.priorQuestions?.length ? { priorQuestions: opts.priorQuestions } : {}), }) - const mcp = createMcpServer({ extraTools: coord.tools, serverName: 'coordination' }) + await coord.ready() + const mcp = createMcpServer({ + extraTools: [...coord.tools, ...(opts.nodeTools ?? [])], + serverName: 'coordination', + }) const host = opts.host ?? '127.0.0.1' const server: Server = createServer((req, res) => { diff --git a/src/runtime/supervise/deadline.ts b/src/runtime/supervise/deadline.ts new file mode 100644 index 00000000..cddf869a --- /dev/null +++ b/src/runtime/supervise/deadline.ts @@ -0,0 +1,110 @@ +import { ValidationError } from '../../errors' +import type { Executor } from './types' + +// Node clamps larger delays to 1 ms instead of waiting, so long budgets must be armed in chunks. +const MAX_TIMER_DELAY_MS = 2_147_483_647 + +export const DEFAULT_SUCCESSFUL_SHUTDOWN_MS = 5_000 +const TEARDOWN_ACKNOWLEDGEMENT_MS = 250 + +/** Arm one wall-clock deadline without keeping the Node.js process alive. */ +export function armDeadlineTimer(delayMs: number, onDeadline: () => void): () => void { + const deadlineAtMs = Date.now() + Math.max(0, delayMs) + let cleared = false + let timer: ReturnType | undefined + + const arm = (): void => { + if (cleared) return + const remainingMs = Math.max(0, deadlineAtMs - Date.now()) + timer = setTimeout( + () => { + if (cleared) return + if (Date.now() >= deadlineAtMs) onDeadline() + else arm() + }, + Math.min(remainingMs, MAX_TIMER_DELAY_MS), + ) + if (typeof timer.unref === 'function') timer.unref() + } + + arm() + return () => { + cleared = true + if (timer !== undefined) clearTimeout(timer) + } +} + +/** + * Resolve a spawned child's duration into one absolute cutoff. An omitted child deadline inherits + * the parent cutoff; an explicit child duration may shorten, but never extend, that cutoff. + */ +export function boundedChildDeadlineAt( + parentDeadlineAtMs: number, + childDeadlineMs: number | undefined, + nowMs: number, +): number | undefined { + const parent = parentDeadlineAtMs > 0 ? parentDeadlineAtMs : undefined + const child = childDeadlineMs === undefined ? undefined : nowMs + childDeadlineMs + if (parent === undefined) return child + if (child === undefined) return parent + return Math.min(parent, child) +} + +/** Invoke teardown once and bound how long the runtime waits for its acknowledgement. A hard + * execution deadline always wins; without one, numeric grace gets a short acknowledgement + * allowance and a brutal kill gets only that allowance. Explicit `infinity` remains unbounded + * only when no execution deadline exists. The losing promise stays observed. */ +export async function teardownExecutor( + executor: Executor, + grace: number | 'brutalKill' | 'infinity', + deadlineAtMs: number | undefined, + now: () => number, +): Promise { + const work = Promise.resolve(executor.teardown(grace)) + + const requestedWaitMs = + grace === 'infinity' + ? undefined + : grace === 'brutalKill' + ? TEARDOWN_ACKNOWLEDGEMENT_MS + : grace + TEARDOWN_ACKNOWLEDGEMENT_MS + // The execution cutoff stops new work; cleanup still gets a small, bounded acknowledgement + // window after that cutoff. Otherwise an already-resolved brutal kill invoked exactly at the + // deadline would be mislabeled as a cleanup failure without receiving one microtask. + const deadlineWaitMs = + deadlineAtMs === undefined + ? undefined + : Math.max(0, deadlineAtMs - now()) + TEARDOWN_ACKNOWLEDGEMENT_MS + const waitMs = + requestedWaitMs === undefined + ? deadlineWaitMs + : deadlineWaitMs === undefined + ? requestedWaitMs + : Math.min(requestedWaitMs, deadlineWaitMs) + + let receipt: { destroyed: boolean } + if (waitMs === undefined) { + receipt = await work + } else if (waitMs <= 0) { + void work.catch(() => undefined) + throw new ValidationError('executor teardown did not acknowledge before its deadline') + } else { + let timer: ReturnType | undefined + const timedOut = new Promise((_resolve, reject) => { + timer = setTimeout( + () => + reject(new ValidationError(`executor teardown did not acknowledge within ${waitMs}ms`)), + waitMs, + ) + }) + try { + receipt = await Promise.race([work, timedOut]) + } finally { + if (timer !== undefined) clearTimeout(timer) + } + } + + if (!receipt.destroyed) { + throw new ValidationError('executor teardown reported destroyed=false') + } +} diff --git a/src/runtime/supervise/driver-executor.ts b/src/runtime/supervise/driver-executor.ts index 6429f164..eadf5846 100644 --- a/src/runtime/supervise/driver-executor.ts +++ b/src/runtime/supervise/driver-executor.ts @@ -6,8 +6,8 @@ * A spawned child resolves through the open registry to an `Executor`; the built-in * executors (router/inline, sandbox, cli) are LEAVES — `execute(task, signal)` runs the * work and settles. This executor is the recursive case: on `execute`, it mounts a NESTED - * `Scope` (the scope hands it the mount via the `nested-scope` seam) over the SAME - * conserved pool + shared journal/blobs + the same open registry, one `depth` deeper, then + * `Scope` (the scope hands it the mount via the `nested-scope` seam) over a child allocation + * carved from its parent reservation, plus shared journal/blobs and the same open registry, then * runs the wrapped driver `Agent.act(task, nestedScope)`. The driver spawns its own * children into that nested scope; each resolves to EITHER a leaf executor (a worker child) * OR this same driver-executor (a driver child) — recursively. So a driver spawns a driver @@ -15,18 +15,16 @@ * * Why this preserves every keystone invariant (the scope owns the sharing; this executor * only runs the driver over what the scope mounts): - * - Conserved budget: the nested scope reserves from the SAME `BudgetPool` the root owns - * (the scope mounts it over `args.pool`), so `Σk` is conserved ACROSS depth by - * construction — a deep tree cannot overspend the root ceiling (reserve-on-spawn fails - * closed at any depth). + * - Conserved budget: the parent reserves the driver's whole allocation once. The nested scope + * partitions only that allocation, and its actual child work + driver inference reconcile the + * parent ticket once at settle. A deep tree cannot overspend or double-charge the root total. * - Journal: the nested scope writes to its OWN tree key (`${journalRoot}/${nodeId}`) so * its cursor `seq`s never collide with the parent's in the per-tree uniqueness guard, * while every nested tree shares the one `SpawnJournal` — the whole recursion is one * journal, queryable tree by tree. - * - Settlement bubbling: the driver child settles into its PARENT scope with the conserved - * spend summed off its nested tree's settled events, so the parent's pool reconcile + - * the supervisor's `spentTotal` see the whole sub-tree's spend rolled up — settlements - * bubble to the root. + * - Settlement bubbling: the driver child settles into its PARENT scope with child work and + * driver inference kept separate in the journal, while their sum reconciles the one parent + * reservation. The supervisor sees the whole sub-tree exactly once. * - Depth ceiling: the nested scope runs at `depth+1`, so the supervisor's `maxDepth` * (paired with the conserved pool per R3) fails a spawn closed once the recursion is too * deep — exactly as it does for a flat tree. @@ -38,12 +36,24 @@ * @experimental */ +import type { AgentProfile } from '@tangle-network/agent-interface' import { ValidationError } from '../../errors' -import { type NestedScopeSeam, nestedScopeSeamKey } from './scope' +import { + attestRuntimeOwnedDeferredExecutor, + runtimeOwnedScopeOwnerRuntime, +} from './materialization' +import { + finalizeScopeOwnerMaterialization, + type NestedScopeSeam, + nestedScopeSeamKey, +} from './scope' import type { Agent, + AgentExecutionRef, AgentSpec, DefaultVerdict, + Executor, + ExecutorAccounting, ExecutorContext, ExecutorFactory, ExecutorRegistry, @@ -62,6 +72,7 @@ const driverRole = 'driver' /** A driver child's spec carries the `Agent` to run inside the nested scope. */ interface DriverSpec extends AgentSpec { + readonly driverRuntime: typeof driverRuntime readonly driver: Agent /** The shared journal the nested tree is one tree key inside (so the executor can * begin its nested tree + sum its spend off the same record). */ @@ -76,13 +87,21 @@ interface DriverSpec extends AgentSpec { * called directly — a driver child runs THROUGH its nested-scope executor, never as a root. */ export function driverChild( - name: string, + profileOrName: AgentProfile | string, driver: Agent, journal: SpawnJournal, + execution?: AgentExecutionRef, ): Agent { + const profile: AgentProfile = + typeof profileOrName === 'string' + ? { name: profileOrName, metadata: { role: driverRole } } + : profileOrName + const name = profile.name ?? driver.name const spec: DriverSpec = { - profile: { name, metadata: { role: driverRole } } as AgentSpec['profile'], + profile, harness: null, + ...(execution ? { execution } : {}), + driverRuntime, driver: driver as Agent, journal, } @@ -99,8 +118,7 @@ export function driverChild( /** True when a spec is a driver child (carries the role marker + a driver Agent). */ export function isDriverSpec(spec: AgentSpec): spec is DriverSpec { - const role = (spec.profile.metadata as { role?: unknown } | undefined)?.role - if (role !== driverRole) return false + if ((spec as { driverRuntime?: unknown }).driverRuntime !== driverRuntime) return false const driver = (spec as { driver?: unknown }).driver if (!isAgent(driver)) { throw new ValidationError( @@ -114,7 +132,7 @@ export function isDriverSpec(spec: AgentSpec): spec is DriverSpec { * The recursive driver-executor factory. `withDriverExecutor` routes a child marked * `role: 'driver'` here; any other child resolves to a leaf built-in. On `execute`, it * reads the `nested-scope` seam the SCOPE seeded, mounts a nested `Scope` one `depth` - * deeper over the shared pool/journal/blobs/registry, runs the driver + * deeper over the driver's reserved child pool plus shared journal/blobs/registry, runs the driver * `Agent.act(task, nestedScope)`, and reports the conserved spend summed off the nested * tree's settled events — so the parent scope's reconcile rolls the whole sub-tree's spend * into the conserved total. @@ -138,30 +156,58 @@ export const driverExecutorFactory: ExecutorFactory = (spec, ctx) => { // on BOTH the success AND crash paths (metered events are durable in the nested tree regardless), // so a sub-driver that crashes mid-run still re-homes its partial inference — pool + journal agree. let meteredSpend: Spend | undefined + let accounting: ExecutorAccounting | undefined + let active: + | { + readonly controller: AbortController + readonly scope: Scope + close?: Promise + } + | undefined - return { + const closeActive = (reason: string): Promise => { + if (active === undefined) return Promise.resolve() + active.close ??= closeNestedScope(active.scope, active.controller, reason) + return active.close + } + + const executor: Executor = { runtime: driverRuntime, async execute(task, signal): Promise> { // The nested tree key namespaces this driver's children inside the ONE shared // journal, so its cursor seqs never collide with the parent's per-tree guard. - const nestedRoot = nestedTreeKey(seam, journal) + const nestedRoot = nestedTreeKey(seam) await journal.beginTree(nestedRoot, new Date(0).toISOString()) - const nestedScope: Scope = seam.mount(nestedRoot, signal) + const controller = new AbortController() + const onParentAbort = () => controller.abort(signal.reason) + if (signal.aborted) controller.abort(signal.reason) + else signal.addEventListener('abort', onParentAbort, { once: true }) + const nestedScope: Scope = seam.mount(nestedRoot, controller.signal) + active = { controller, scope: nestedScope } try { // Run the driver. Its `act` spawns children into the nested scope and reacts via // `scope.next()`; a thrown `act` propagates so the PARENT scope types it into a down. - const out = await driver.act(task, nestedScope) + const out = await driverActAbortable(() => driver.act(task, nestedScope), controller.signal) + await finalizeScopeOwnerMaterialization(nestedScope) - // Read the nested tree's events ONCE. Two roll-ups, kept separate so the conserved invariant - // is not double-charged: - // - `spent` = settled child WORK → reconciled against THIS driver's reservation (as before). - // - `metered` = the nested subtree's driver INFERENCE → re-homed by the parent scope as a - // `metered` event, NOT reconciled (already pool-debited live via `observe`). + // A driver may choose its answer while descendants still run. Its allocation is not + // refundable until every descendant has stopped and reached a terminal journal record. + await closeActive('driver completed') + + // Read the nested tree's events ONCE. Keep two journal categories while reconciling their + // sum against this driver's one parent reservation: + // - `spent` = settled child WORK, written on this driver's settlement. + // - `metered` = nested driver INFERENCE, re-homed as a separate metered event. const events = await loadTreeEvents(journal, nestedRoot) const settled = events.filter(isSettled) + const childWork = sumSpend(settled) meteredSpend = nonZeroOrUndef(sumMetered(events)) + accounting = { + reported: childWork, + reservation: addSpend(childWork, meteredSpend ?? zeroSpend()), + } // Completion-oracle propagation: a driver "delivered" iff at least one of its DIRECT // children settled `valid` (the child its keep-best finalize returns). Deriving the // driver child's verdict this way composes delivery UP the recursion — a sub-driver is @@ -171,27 +217,35 @@ export const driverExecutorFactory: ExecutorFactory = (spec, ctx) => { artifact = { outRef: `${driverRuntime}:${nestedRoot}`, out, - spent: sumSpend(settled), + spent: childWork, ...(verdict ? { verdict } : {}), } return artifact } catch (err) { + await finalizeScopeOwnerMaterialization(active?.scope ?? nestedScope).catch(() => undefined) + await closeActive('driver stopped') // Crash mid-run: the nested tree still holds the durable `metered` events the sub-driver // already wrote (pool already debited them). Cache them so the parent's down-path re-home // lands the partial inference and the two ledgers stay in agreement. A missing tree must // not mask the original error. - meteredSpend = await safeSumMetered(journal, nestedRoot) + const partial = await safeRollup(journal, nestedRoot) + meteredSpend = partial?.metered + accounting = partial?.accounting throw err + } finally { + signal.removeEventListener('abort', onParentAbort) + active = undefined } }, + accounting(): ExecutorAccounting | undefined { + return accounting + }, metered(): Spend | undefined { return meteredSpend }, - teardown(): Promise<{ destroyed: boolean }> { - // The nested scope's live children are torn down by the driver's own `act` discipline - // (it drains to settlement) and by the parent's abort cascade through `signal`; there - // is no separate box/process to reap here. - return Promise.resolve({ destroyed: true }) + async teardown(): Promise<{ destroyed: boolean }> { + await closeActive('driver executor teardown') + return { destroyed: true } }, resultArtifact(): ExecutorResult { if (!artifact) { @@ -200,6 +254,10 @@ export const driverExecutorFactory: ExecutorFactory = (spec, ctx) => { return artifact }, } + const ownerRuntime = runtimeOwnedScopeOwnerRuntime(driver) + return ownerRuntime === undefined + ? executor + : attestRuntimeOwnedDeferredExecutor(executor, ownerRuntime) } /** @@ -213,8 +271,7 @@ export function withDriverExecutor(base: ExecutorRegistry): ExecutorRegistry { return { register: base.register.bind(base), resolve(spec: AgentSpec) { - const role = (spec.profile.metadata as { role?: unknown } | undefined)?.role - if (role === driverRole && !spec.executor) { + if ((spec as { driverRuntime?: unknown }).driverRuntime === driverRuntime && !spec.executor) { return { succeeded: true as const, value: driverExecutorFactory as ExecutorFactory } } return base.resolve(spec) @@ -224,23 +281,76 @@ export function withDriverExecutor(base: ExecutorRegistry): ExecutorRegistry { // ── Helpers ────────────────────────────────────────────────────────────────────── -/** Mint a unique nested-tree key under the parent's journal root. Uses the parent's - * `journalRoot` + a per-journal monotonic ordinal so two sibling driver trees never - * collide their keys (each driver child mints exactly one nested tree). */ -function nestedTreeKey(seam: NestedScopeSeam, journal: SpawnJournal): string { - return `${seam.journalRoot}/d${nextNestOrdinal(journal)}` +/** Derive the nested-tree key from the parent's durable journal root and this driver child's stable + * node id. Sibling node ids are unique within the parent tree, and replay derives the same key. */ +function nestedTreeKey(seam: NestedScopeSeam): string { + return `${seam.journalRoot}/${seam.nodeId}` } -/** Per-journal monotonic nest counter — keyed on the journal instance so a single run's - * nested-tree keys are unique without a shared module global. */ -const nestCounters = new WeakMap() -function nextNestOrdinal(journal: SpawnJournal): number { - let c = nestCounters.get(journal) - if (!c) { - c = { n: 0 } - nestCounters.set(journal, c) +async function closeNestedScope( + scope: Scope, + controller: AbortController, + reason: string, +): Promise { + if (!controller.signal.aborted) controller.abort(reason) + for (;;) { + const settled = await scope.next() + if (settled === null) break } - return c.n++ + const view = scope.view + if (view.inFlight > 0 || view.waiting > 0) { + throw new ValidationError( + `driverExecutor: nested cleanup left ${view.inFlight} running and ${view.waiting} waiting nodes`, + ) + } +} + +async function driverActAbortable(act: () => Promise, signal: AbortSignal): Promise { + if (signal.aborted) throw driverAbortError(signal) + return await new Promise((resolve, reject) => { + let settled = false + const cleanup = () => signal.removeEventListener('abort', onAbort) + const onAbort = () => { + queueMicrotask(() => { + if (settled) return + settled = true + cleanup() + reject(driverAbortError(signal)) + }) + } + signal.addEventListener('abort', onAbort, { once: true }) + let work: Promise + try { + work = Promise.resolve(act()) + } catch (error) { + cleanup() + reject(error) + return + } + work.then( + (value) => { + if (settled) return + settled = true + cleanup() + resolve(value) + }, + (error) => { + if (settled) return + settled = true + cleanup() + reject(error) + }, + ) + }) +} + +function driverAbortError(signal: AbortSignal): Error { + const reason = signal.reason + const error = new Error( + typeof reason === 'string' && reason.length > 0 ? reason : 'driver aborted', + ) + error.name = 'AbortError' + return error } /** The nested tree's full event list — the one evidence the spend, verdict, AND driver-inference @@ -267,6 +377,7 @@ function sumSpend(settled: ReadonlyArray<{ spent: Spend }>): Spend { total.iterations += ev.spent.iterations total.tokens.input += ev.spent.tokens.input total.tokens.output += ev.spent.tokens.output + if (ev.spent.tokensKnown === false) total.tokensKnown = false total.usd += ev.spent.usd if (ev.spent.usdKnown === false) total.usdKnown = false total.ms += ev.spent.ms @@ -284,6 +395,7 @@ function sumMetered(events: ReadonlyArray): Spend { total.iterations += ev.spend.iterations total.tokens.input += ev.spend.tokens.input total.tokens.output += ev.spend.tokens.output + if (ev.spend.tokensKnown === false) total.tokensKnown = false total.usd += ev.spend.usd if (ev.spend.usdKnown === false) total.usdKnown = false total.ms += ev.spend.ms @@ -291,8 +403,34 @@ function sumMetered(events: ReadonlyArray): Spend { return total } +function zeroSpend(): Spend { + return { iterations: 0, tokens: { input: 0, output: 0 }, usd: 0, ms: 0 } +} + +function addSpend(a: Spend, b: Spend): Spend { + return { + iterations: a.iterations + b.iterations, + tokens: { + input: a.tokens.input + b.tokens.input, + output: a.tokens.output + b.tokens.output, + }, + ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), + usd: a.usd + b.usd, + ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), + ms: a.ms + b.ms, + } +} + function isNonZeroSpend(s: Spend): boolean { - return s.iterations > 0 || s.tokens.input > 0 || s.tokens.output > 0 || s.usd > 0 || s.ms > 0 + return ( + s.iterations > 0 || + s.tokens.input > 0 || + s.tokens.output > 0 || + s.tokensKnown === false || + s.usd > 0 || + s.usdKnown === false || + s.ms > 0 + ) } /** A spend, or `undefined` when it is all-zero — so `metered()` returns undefined for a driver @@ -303,12 +441,23 @@ function nonZeroOrUndef(s: Spend): Spend | undefined { /** Sum the nested tree's metered events, tolerating a missing tree (a crash before `beginTree` * landed) — never throw here, or it would mask the original `act` error on the crash path. */ -async function safeSumMetered( +async function safeRollup( journal: SpawnJournal, nestedRoot: string, -): Promise { +): Promise< + { readonly metered: Spend | undefined; readonly accounting: ExecutorAccounting } | undefined +> { try { - return nonZeroOrUndef(sumMetered(await loadTreeEvents(journal, nestedRoot))) + const events = await loadTreeEvents(journal, nestedRoot) + const childWork = sumSpend(events.filter(isSettled)) + const metered = nonZeroOrUndef(sumMetered(events)) + return { + metered, + accounting: { + reported: childWork, + reservation: addSpend(childWork, metered ?? zeroSpend()), + }, + } } catch { return undefined } diff --git a/src/runtime/supervise/event-bus.ts b/src/runtime/supervise/event-bus.ts index 5a7be78c..cc81c620 100644 --- a/src/runtime/supervise/event-bus.ts +++ b/src/runtime/supervise/event-bus.ts @@ -12,8 +12,8 @@ * queued settles/findings so the driver sees it first; ties resolve FIFO by publish order. * * Observability is first-class (A++): every event is stamped with a monotonic `seq` and wall-clock - * `at`, the full ordered `history()` is retained as an audit/replay trail, and `stats()` exposes - * published/pulled counts by kind. Subscribers receive the stamped record, not a bare event. + * `at`, the full ordered `history()` is retained as current-process audit evidence, and `stats()` + * exposes published/pulled counts by kind. Subscribers receive the stamped record, not a bare event. * * The interface is transport-agnostic on purpose. Same box → this in-process queue. Cross box → * the SAME publish/pull/subscribe surface backed by a durable mailbox on the parent's box (children @@ -55,8 +55,9 @@ export interface BusStats { } export interface EventBus { - /** Stamp + queue the event, then deliver the stamped record to every subscriber in order. - * Returns the stamped record. */ + /** Stamp the event, await every subscriber in order, then make it pull-visible. A subscriber + * failure leaves the event invisible and retrying the SAME event object reuses the exact stamp. + * This lets an awaited product observer commit its record before a supervisor can consume it. */ publish(event: E, opts?: PublishOptions): Promise> /** Remove and return the highest-priority QUEUED event whose type is in `kinds` (any if omitted), * ties broken FIFO by `seq`; `undefined` when nothing matches. */ @@ -66,7 +67,7 @@ export interface EventBus { subscribe(handler: (record: BusRecord) => void | Promise): () => void /** Count of queued, not-yet-pulled events (filtered by `kinds` when given). */ pending(kinds?: ReadonlyArray): number - /** The full ordered log of every event ever published (the audit/replay trail). */ + /** The full ordered log of every event published in this process (audit evidence, not replay). */ history(): ReadonlyArray> /** Throughput counters for observability dashboards. */ stats(): BusStats @@ -78,7 +79,12 @@ export function createEventBus(now: () => number = Date.now) const log: BusRecord[] = [] const subscribers: Array<(record: BusRecord) => void | Promise> = [] const byKind: Record = {} + // A failed publication is staged, not published. Coordination retains and retries the same event + // object, so the exact BusRecord survives a lost acknowledgement and downstream idempotency keys + // do not change between attempts. + const staged = new WeakMap>() let seq = 0 + let published = 0 let pulled = 0 const matches = (r: BusRecord, kinds?: ReadonlyArray) => @@ -102,14 +108,19 @@ export function createEventBus(now: () => number = Date.now) return { async publish(event, opts) { - const record: BusRecord = { seq: seq++, at: now(), priority: opts?.priority ?? 0, event } + const record = + staged.get(event) ?? + ({ seq: seq++, at: now(), priority: opts?.priority ?? 0, event } satisfies BusRecord) + staged.set(event, record) + // Sequential, not Promise.all: transaction observers must see causal order. Nothing enters + // the pull queue or successful history until all observers acknowledge the record. + for (const handler of subscribers) await handler(record) + staged.delete(event) // Record-only events (the down-leg) skip the pull queue but still hit the log + subscribers. if (opts?.queue !== false) queue.push(record) log.push(record) + published += 1 byKind[event.type] = (byKind[event.type] ?? 0) + 1 - // Sequential, not Promise.all: a subscriber that steers off this event must observe a - // consistent order, and a throwing subscriber must not silently drop siblings' delivery. - for (const handler of subscribers) await handler(record) return record }, pull(kinds) { @@ -132,7 +143,7 @@ export function createEventBus(now: () => number = Date.now) return log }, stats() { - return { published: seq, pulled, byKind: { ...byKind } } + return { published, pulled, byKind: { ...byKind } } }, } } diff --git a/src/runtime/supervise/run-context.ts b/src/runtime/supervise/run-context.ts index e46c3da3..e86ea1ec 100644 --- a/src/runtime/supervise/run-context.ts +++ b/src/runtime/supervise/run-context.ts @@ -61,10 +61,11 @@ export interface InMemoryRunContext { */ readonly resume?: boolean /** - * Present only on a DURABLE context: the coordination side-log (questions + analyst findings — - * the bus messages the spawn journal does not record). `supervise({ runDir })` appends to it as - * they publish and replays it on resume, so a restarted coordinator keeps them. In-memory - * contexts have none: nothing outlives the process to replay into. + * Present only on a DURABLE context: the coordination side-log stores questions, analyst + * findings, answer decisions, and authorized continuation receipts that the spawn journal does + * not own. `supervise({ runDir })` appends them as they publish and loads them on resume. + * Continuation receipts are evidence and are never auto-delivered to a replacement worker. + * In-memory contexts have none: nothing outlives the process. */ readonly coordinationLog?: CoordinationLog } @@ -94,8 +95,9 @@ export function createInMemoryRunContext(opts: InMemoryRunContextOptions = {}): * back on `Scope.resume` (rehydrated by `replaySpawnTree`) instead of being re-executed. * * Layout: `${dir}/spawn-journal.jsonl` (one JSONL record per event), `${dir}/blobs/` (one - * content-addressed JSON file per settled result), and `${dir}/coordination-log.jsonl` (questions - * + findings, replayed into a resumed driver). The directory is created on first write. + * content-addressed JSON file per settled result), and `${dir}/coordination-log.jsonl` + * (questions, findings, answer decisions, and authorized continuation receipts retained as + * evidence). The directory is created on first write. * * Opt-in by construction — `createInMemoryRunContext()` is unchanged and stays the default, so no * existing consumer writes to disk or resumes unless it asks for this. diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 95fb0998..27f771b8 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -30,6 +30,11 @@ import { request as httpRequest } from 'node:http' import { request as httpsRequest } from 'node:https' import { Readable } from 'node:stream' import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' +import { + type AgentProfile, + agentProfileSchema, + mergeAgentProfiles, +} from '@tangle-network/agent-interface' import type { BackendType, SandboxEvent } from '@tangle-network/sandbox' import { ValidationError } from '../../errors' import type { LocalHarness } from '../../mcp/local-harness' @@ -65,9 +70,11 @@ import type { } from '../types' import { zeroTokenUsage } from '../util' import { createInbox, type Inbox } from './inbox' +import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' import { PI_RUNTIME, type PiSeam, piExecutor } from './pi-executor' import type { ExecutorProgress } from './progress' import { createSteerableSandboxSession, type SandboxSteeringOptions } from './sandbox-session' +import { detachedSnapshot } from './snapshot' import type { TraceSource } from './trace-source' import type { AgentSpec, @@ -139,7 +146,9 @@ export interface CliSeam { /** * cli-worktree seam. A supervisor-authored `AgentProfile` driving a local coding-harness CLI * (claude / codex / opencode) on its own git worktree — the leaf `createWorktreeCliExecutor` - * named as data. `harness` + `repoRoot` + `taskPrompt` are required; the authored + * named as data. `harness` + `repoRoot` are required; the task comes from `Executor.execute`. + * `taskPrompt` remains an optional direct-call fallback for callers that execute with `undefined`. + * The authored * `profile.prompt.systemPrompt` + `profile.model.default` reach the harness via the §1.5 * `harnessInvocation` mapper. Everything else mirrors `WorktreeCliExecutorOptions`. */ @@ -147,7 +156,7 @@ export interface CliWorktreeSeam { repoRoot: string /** Local CLI harness transport. Omit when `bridge` is set. */ harness?: LocalHarness - taskPrompt: string + taskPrompt?: string runId?: string baseRef?: string harnessTimeoutMs?: number @@ -174,7 +183,8 @@ export interface CliWorktreeBridgeSeam { bridgeBearer: string /** Bridge model/harness id. Defaults to the profile's model hint when omitted. */ model?: string - agentProfile?: Record + /** Canonical profile overlay merged over the spawned profile. */ + agentProfile?: AgentProfile timeoutMs?: number /** Stable cli-bridge session id. Defaults to `bridge-worktree-${runId}`. */ sessionId?: string @@ -189,19 +199,20 @@ export interface CliWorktreeBridgeSeam { * forwarded verbatim per request — how an arm disables native tools or injects * a provider search MCP. * - * The executor opens a RESUMABLE cli-bridge session — structurally identical to the - * sandbox executor's persistent box, just local. `sessionId` is the stable - * caller-owned id cli-bridge maps to the harness's internal conversation id; a - * follow-up steer/resume on the SAME id continues the SAME harness session (opencode - * `-s`, claude `--resume`, …). Omit it and the executor mints a stable one per spawn. + * The executor opens a resumable cli-bridge session. `sessionId` identifies the + * harness conversation across turns; each turn also receives its own durable run id. + * A dropped HTTP reader reattaches to that exact run and explicit cancel is the only + * operation allowed to stop it. Omit `sessionId` and the executor mints one per spawn. */ export interface BridgeSeam { bridgeUrl: string bridgeBearer: string - model: string + /** Fallback bridge wire id. A spawned profile may select its own harness and model. */ + model?: string /** Optional working directory forwarded to cli-bridge and persisted with the session. */ cwd?: string - agentProfile?: Record + /** Canonical profile overlay merged over the spawned profile. */ + agentProfile?: AgentProfile timeoutMs?: number /** Stable, caller-owned cli-bridge session id for harness-side resume. Defaults * to a freshly minted per-spawn id so each worker is its own resumable session. */ @@ -265,7 +276,7 @@ function zeroSpend(): Spend { */ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { const seam = readSeam(ctx, routerSeamKey, 'router/inline') - const model = seam.model ?? spec.profile.model?.default + const model = spec.profile.model?.default ?? seam.model if (!model) { throw new ValidationError( 'routerInlineExecutor: no model — set RouterSeam.model or AgentProfile.model.default', @@ -283,39 +294,65 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { if (!ctx.signal.aborted) ctx.signal.addEventListener('abort', abortIfSignalled, { once: true }) let artifact: ExecutorResult | undefined - - return { - runtime: 'router' as Runtime, - async execute(task, signal): Promise> { - const messages = taskToMessages(task, spec) - const started = Date.now() - const linked = linkSignals(signal, controller.signal) - const r = await routerChatWithUsage( - { routerBaseUrl: seam.routerBaseUrl, routerKey: seam.routerKey, model }, - messages, - linked ? { signal: linked } : {}, - ) - const spent: Spend = { - iterations: 1, - tokens: r.usage ? { input: r.usage.input, output: r.usage.output } : zeroTokenUsage(), - usd: r.costUsd ?? 0, - ms: Date.now() - started, - } - const out = { content: r.content } as unknown - artifact = { outRef: contentRef('router', { model, content: r.content }), out, spent } - return artifact + const executionId = ctx.node?.nodeId ?? `router-request-${randomUUID()}` + const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(executionId) + + return attestRuntimeOwnedExecutor( + { + runtime: 'router' as Runtime, + async execute(task, signal): Promise> { + const messages = taskToMessages(task, spec) + const started = Date.now() + const linked = linkSignals(signal, controller.signal) + const r = await routerChatWithUsage( + { routerBaseUrl: seam.routerBaseUrl, routerKey: seam.routerKey, model }, + messages, + linked ? { signal: linked } : {}, + ) + const spent: Spend = { + iterations: 1, + tokens: r.usage ? { input: r.usage.input, output: r.usage.output } : zeroTokenUsage(), + usd: r.costUsd ?? 0, + ...(r.usage ? {} : { tokensKnown: false }), + ...(r.costUsd === undefined ? { usdKnown: false } : {}), + ms: Date.now() - started, + } + const out = { content: r.content } as unknown + artifact = { outRef: contentRef('router', { model, content: r.content }), out, spent } + return artifact + }, + teardown(_grace): Promise<{ destroyed: boolean }> { + controller.abort() + return Promise.resolve({ destroyed: true }) + }, + resultArtifact() { + if (!artifact) { + throw new ValidationError('routerInlineExecutor: resultArtifact() read before execute()') + } + return { ...artifact, spent: artifact.spent } + }, }, - teardown(_grace): Promise<{ destroyed: boolean }> { - controller.abort() - return Promise.resolve({ destroyed: true }) + { + effectiveProfile: spec.profile, + backend: 'router', + model: { status: 'known', id: model }, + execution: { + kind: 'request', + id: executionId, + }, + materializer: 'router-prompt-model', + plan: { kind: 'openai-chat-completion', model }, }, - resultArtifact() { - if (!artifact) { - throw new ValidationError('routerInlineExecutor: resultArtifact() read before execute()') - } - return { ...artifact, spent: artifact.spent } + { + attemptId, + binding: { + endpoint: seam.routerBaseUrl, + executionId, + model, + }, + descriptor: { kind: 'router-request', transport: 'http', backend: 'router' }, }, - } + ) } export type { ToolSpec } @@ -374,7 +411,7 @@ interface RouterToolsResponse { */ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) => { const seam = readSeam(ctx, routerToolsSeamKey, 'router-tools') - const model = seam.model ?? spec.profile.model?.default + const model = spec.profile.model?.default ?? seam.model if (!model) { throw new ValidationError( 'routerToolsInlineExecutor: no model — set RouterToolsSeam.model or AgentProfile.model.default', @@ -398,174 +435,212 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = const inbox = createInbox() let artifact: ExecutorResult | undefined + const executionId = ctx.node?.nodeId ?? `router-tools-run-${randomUUID()}` + const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(executionId) - return { - runtime: 'router' as Runtime, - deliver: (m) => inbox.deliver(m), - async execute(task, signal): Promise> { - const started = Date.now() - const messages: Array> = [ - ...(taskToMessages(task, spec) as Array>), - ] - const tokens = zeroTokenUsage() - let turns = 0 - let lastText = '' - // Fold any queued down-messages into the conversation as one operator turn (the boundary flush). - const flush = () => { - const pending = inbox.drain() - if (pending.length) messages.push({ role: 'user', content: inbox.fold(pending) }) - return pending.length > 0 - } - - // The external abort sources (caller signal + executor teardown), merged ONCE — so we don't - // re-register listeners on these long-lived signals every turn. - const external = mergeAbortSignals(signal, controller.signal) - - for (let t = 0; t < maxTurns; t += 1) { - // QUEUED messages flush at the step boundary, before this turn's inference. - flush() - // A forceful (interrupt) message aborts THIS turn so the worker re-plans immediately. The - // per-turn controller fires on `external` OR a fresh interrupt; its listener on `external` is - // removed after the turn (`cleanup`) so nothing accumulates across turns. - const interruptSig = inbox.freshInterrupt() - const turnController = new AbortController() - const abortTurn = () => turnController.abort() - if (external.aborted) turnController.abort() - else external.addEventListener('abort', abortTurn) - interruptSig.addEventListener('abort', abortTurn, { once: true }) - const cleanup = () => external.removeEventListener('abort', abortTurn) - let res: Response - try { - res = await fetch(`${seam.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${seam.routerKey}`, - }, - body: JSON.stringify({ - model, - messages, - tools: seam.tools, - tool_choice: 'auto', - temperature: 0.2, - }), - signal: turnController.signal, - }) - } catch (e) { - cleanup() - // Re-plan ONLY when a forceful inbox message aborted this turn (a real AbortError, with the - // interrupt — not the external teardown/budget signal). The re-planned turn still consumes a - // loop slot (so interrupt spam is bounded by maxTurns, not a hang) but does not bill a turn. - // Any other error — incl. a network fault coincident with an interrupt — is fatal: rethrow. - const interruptAbort = - e instanceof DOMException && - e.name === 'AbortError' && - interruptSig.aborted && - !signal.aborted && - !controller.signal.aborted - if (interruptAbort) continue - throw e - } - cleanup() - // The inference completed — count the turn now (an interrupted, re-planned turn doesn't bill). - turns += 1 - if (!res.ok) { - throw new ValidationError( - `routerToolsInlineExecutor: router ${res.status}: ${(await res.text()).slice(0, 200)}`, - ) - } - const data = (await res.json()) as RouterToolsResponse - const u = data.usage - if (u && typeof u.prompt_tokens === 'number' && typeof u.completion_tokens === 'number') { - tokens.input += u.prompt_tokens - tokens.output += u.completion_tokens - } - const msg = data.choices?.[0]?.message - if (msg?.content) lastText = msg.content - const toolCalls = msg?.tool_calls ?? [] - if (toolCalls.length === 0) { - // Before settling, flush once more — a worker may not finish while a steer/answer it never - // read is still pending. If anything flushed, keep going; otherwise it is truly done. - if (flush()) continue - break + return attestRuntimeOwnedExecutor( + { + runtime: 'router' as Runtime, + deliver: (m) => inbox.deliver(m), + async execute(task, signal): Promise> { + const started = Date.now() + const messages: Array> = [ + ...(taskToMessages(task, spec) as Array>), + ] + const tokens = zeroTokenUsage() + let tokensKnown = true + let turns = 0 + let lastText = '' + // Fold any queued down-messages into the conversation as one operator turn (the boundary flush). + const flush = () => { + const pending = inbox.drain() + if (pending.length) messages.push({ role: 'user', content: inbox.fold(pending) }) + return pending.length > 0 } - // Record the assistant turn verbatim, then run each call on the host and - // fold the result back as a `tool` message for the next turn. - messages.push({ - role: 'assistant', - content: msg?.content ?? '', - tool_calls: toolCalls.map((tc, i) => ({ - id: tc.id ?? `call_${i}`, - type: 'function', - function: { name: tc.function?.name ?? '', arguments: tc.function?.arguments ?? '{}' }, - })), - }) - for (let i = 0; i < toolCalls.length; i += 1) { - const tc = toolCalls[i] - const id = tc?.id ?? `call_${i}` - let args: Record = {} + // The external abort sources (caller signal + executor teardown), merged ONCE — so we don't + // re-register listeners on these long-lived signals every turn. + const external = mergeAbortSignals(signal, controller.signal) + + for (let t = 0; t < maxTurns; t += 1) { + // QUEUED messages flush at the step boundary, before this turn's inference. + flush() + // A forceful (interrupt) message aborts THIS turn so the worker re-plans immediately. The + // per-turn controller fires on `external` OR a fresh interrupt; its listener on `external` is + // removed after the turn (`cleanup`) so nothing accumulates across turns. + const interruptSig = inbox.freshInterrupt() + const turnController = new AbortController() + const abortTurn = () => turnController.abort() + if (external.aborted) turnController.abort() + else external.addEventListener('abort', abortTurn) + interruptSig.addEventListener('abort', abortTurn, { once: true }) + const cleanup = () => external.removeEventListener('abort', abortTurn) + let res: Response try { - args = JSON.parse(tc?.function?.arguments ?? '{}') as Record - } catch { - // Malformed args are a real outcome, not an infra fault — feed the error - // back so the model can correct, rather than aborting the whole loop. - messages.push({ - role: 'tool', - tool_call_id: id, - content: 'error: tool arguments were not valid JSON', + res = await fetch(`${seam.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${seam.routerKey}`, + }, + body: JSON.stringify({ + model, + messages, + tools: seam.tools, + tool_choice: 'auto', + temperature: 0.2, + }), + signal: turnController.signal, }) - continue - } - const toolName = tc?.function?.name ?? '' - let result: string - let status: 'ok' | 'error' = 'ok' - const toolStartedAt = Date.now() - try { - result = await seam.executeToolCall(toolName, args, task) } catch (e) { - status = 'error' - result = `error: ${e instanceof Error ? e.message : String(e)}` + cleanup() + // Re-plan ONLY when a forceful inbox message aborted this turn (a real AbortError, with the + // interrupt — not the external teardown/budget signal). The re-planned turn still consumes a + // loop slot (so interrupt spam is bounded by maxTurns, not a hang) but does not bill a turn. + // Any other error — incl. a network fault coincident with an interrupt — is fatal: rethrow. + const interruptAbort = + e instanceof DOMException && + e.name === 'AbortError' && + interruptSig.aborted && + !signal.aborted && + !controller.signal.aborted + if (interruptAbort) continue + throw e } - const toolEndedAt = Date.now() - messages.push({ role: 'tool', tool_call_id: id, content: result }) - // Feed the online detector pipe (stuck-loop / error-streak) — a worker repeating the same - // call or hammering errors is caught mid-run, not only at settle. This is an observability - // side-channel: a throwing monitor must never crash the production inference loop. - try { - seam.onToolStep?.({ - toolName, - args, - status, - startedAt: toolStartedAt, - endedAt: toolEndedAt, - durationMs: toolEndedAt - toolStartedAt, - }) - } catch { - // ignore — monitoring must not break the worker + cleanup() + // The inference completed — count the turn now (an interrupted, re-planned turn doesn't bill). + turns += 1 + if (!res.ok) { + throw new ValidationError( + `routerToolsInlineExecutor: router ${res.status}: ${(await res.text()).slice(0, 200)}`, + ) + } + const data = (await res.json()) as RouterToolsResponse + const u = data.usage + if (u && typeof u.prompt_tokens === 'number' && typeof u.completion_tokens === 'number') { + tokens.input += u.prompt_tokens + tokens.output += u.completion_tokens + } else { + tokensKnown = false + } + const msg = data.choices?.[0]?.message + if (msg?.content) lastText = msg.content + const toolCalls = msg?.tool_calls ?? [] + if (toolCalls.length === 0) { + // Before settling, flush once more — a worker may not finish while a steer/answer it never + // read is still pending. If anything flushed, keep going; otherwise it is truly done. + if (flush()) continue + break + } + + // Record the assistant turn verbatim, then run each call on the host and + // fold the result back as a `tool` message for the next turn. + messages.push({ + role: 'assistant', + content: msg?.content ?? '', + tool_calls: toolCalls.map((tc, i) => ({ + id: tc.id ?? `call_${i}`, + type: 'function', + function: { + name: tc.function?.name ?? '', + arguments: tc.function?.arguments ?? '{}', + }, + })), + }) + for (let i = 0; i < toolCalls.length; i += 1) { + const tc = toolCalls[i] + const id = tc?.id ?? `call_${i}` + let args: Record = {} + try { + args = JSON.parse(tc?.function?.arguments ?? '{}') as Record + } catch { + // Malformed args are a real outcome, not an infra fault — feed the error + // back so the model can correct, rather than aborting the whole loop. + messages.push({ + role: 'tool', + tool_call_id: id, + content: 'error: tool arguments were not valid JSON', + }) + continue + } + const toolName = tc?.function?.name ?? '' + let result: string + let status: 'ok' | 'error' = 'ok' + const toolStartedAt = Date.now() + try { + result = await seam.executeToolCall(toolName, args, task) + } catch (e) { + status = 'error' + result = `error: ${e instanceof Error ? e.message : String(e)}` + } + const toolEndedAt = Date.now() + messages.push({ role: 'tool', tool_call_id: id, content: result }) + // Feed the online detector pipe (stuck-loop / error-streak) — a worker repeating the same + // call or hammering errors is caught mid-run, not only at settle. This is an observability + // side-channel: a throwing monitor must never crash the production inference loop. + try { + seam.onToolStep?.({ + toolName, + args, + status, + startedAt: toolStartedAt, + endedAt: toolEndedAt, + durationMs: toolEndedAt - toolStartedAt, + }) + } catch { + // ignore — monitoring must not break the worker + } } } - } - const usd = isModelPriced(model) ? estimateCost(tokens.input, tokens.output, model) : 0 - const spent: Spend = { iterations: turns, tokens, usd, ms: Date.now() - started } - const out = { content: lastText } as unknown - artifact = { outRef: contentRef('router-tools', { model, content: lastText }), out, spent } - return artifact + const priced = isModelPriced(model) + const usd = priced ? estimateCost(tokens.input, tokens.output, model) : 0 + const spent: Spend = { + iterations: turns, + tokens, + ...(tokensKnown ? {} : { tokensKnown: false }), + usd, + ...(!priced || !tokensKnown ? { usdKnown: false } : {}), + ms: Date.now() - started, + } + const out = { content: lastText } as unknown + artifact = { outRef: contentRef('router-tools', { model, content: lastText }), out, spent } + return artifact + }, + teardown(_grace): Promise<{ destroyed: boolean }> { + controller.abort() + return Promise.resolve({ destroyed: true }) + }, + resultArtifact() { + if (!artifact) { + throw new ValidationError( + 'routerToolsInlineExecutor: resultArtifact() read before execute()', + ) + } + return { ...artifact, spent: artifact.spent } + }, }, - teardown(_grace): Promise<{ destroyed: boolean }> { - controller.abort() - return Promise.resolve({ destroyed: true }) + { + effectiveProfile: spec.profile, + backend: 'router-tools', + model: { status: 'known', id: model }, + execution: { + kind: 'run', + id: executionId, + }, + materializer: 'router-tools-prompt-model', + plan: { kind: 'openai-tool-loop', model, maxTurns, tools: seam.tools }, }, - resultArtifact() { - if (!artifact) { - throw new ValidationError( - 'routerToolsInlineExecutor: resultArtifact() read before execute()', - ) - } - return { ...artifact, spent: artifact.spent } + { + attemptId, + binding: { + endpoint: seam.routerBaseUrl, + executionId, + model, + }, + descriptor: { kind: 'router-tool-loop', transport: 'http', backend: 'router-tools' }, }, - } + ) } // ── sandbox executor (harness is a BackendType) ──────────────────────────────── @@ -604,6 +679,35 @@ export const sandboxExecutor: ExecutorFactory = (spec, ctx) => { if (!ctx.signal.aborted) ctx.signal.addEventListener('abort', abortIfSignalled, { once: true }) let artifact: ExecutorResult | undefined + const executionId = ctx.node?.nodeId ?? `sandbox-run-${randomUUID()}` + const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(executionId) + const sandboxMaterialization = { + effectiveProfile: spec.profile, + backend: harness, + model: spec.profile.model?.default + ? ({ status: 'known', id: spec.profile.model.default } as const) + : ({ status: 'unknown', reason: 'sandbox harness selected its default model' } as const), + execution: { + kind: 'run', + id: executionId, + }, + materializer: 'sandbox-agent-profile', + plan: { + kind: 'sandbox-agent-rounds', + harness, + maxIterations, + steering: seam.steering !== undefined, + }, + } + const sandboxBinding = { + attemptId, + binding: { + executionId, + harness, + model: spec.profile.model?.default ?? null, + }, + descriptor: { kind: 'sandbox-run', transport: 'sandbox', backend: harness }, + } // STEERABLE mode (opt-in): the worker becomes a multi-turn session on one box, with a real // inbox, so a driver's steer has a turn boundary to be folded into. This is the path that @@ -621,29 +725,33 @@ export const sandboxExecutor: ExecutorFactory = (spec, ctx) => { ...(seam.loopCtx ? { loopCtx: seam.loopCtx } : {}), contentRef, }) - return { - runtime: 'sandbox' as Runtime, - deliver: (m) => inbox.deliver(m), - progress: (): ExecutorProgress => session.progress(), - traceSource: (): TraceSource => session.traceSource(), - execute(task, signal): AsyncIterable { - return session.stream(task, signal) - }, - async teardown(_grace): Promise<{ destroyed: boolean }> { - controller.abort() - await session.teardown() - return { destroyed: true } - }, - resultArtifact() { - const a = session.artifact() - if (!a) { - throw new ValidationError( - 'sandboxExecutor(steering): resultArtifact() read before stream drained', - ) - } - return a + return attestRuntimeOwnedExecutor( + { + runtime: 'sandbox' as Runtime, + deliver: (m) => inbox.deliver(m), + progress: (): ExecutorProgress => session.progress(), + traceSource: (): TraceSource => session.traceSource(), + execute(task, signal): AsyncIterable { + return session.stream(task, signal) + }, + async teardown(_grace): Promise<{ destroyed: boolean }> { + controller.abort() + await session.teardown() + return { destroyed: true } + }, + resultArtifact() { + const a = session.artifact() + if (!a) { + throw new ValidationError( + 'sandboxExecutor(steering): resultArtifact() read before stream drained', + ) + } + return a + }, }, - } + sandboxMaterialization, + sandboxBinding, + ) } // The leaf runs an opaque, self-parallelizing coding harness; the loop just @@ -655,38 +763,42 @@ export const sandboxExecutor: ExecutorFactory = (spec, ctx) => { } const driver = singleShotDriver(maxIterations) - return { - runtime: 'sandbox' as Runtime, - execute(task, signal): AsyncIterable { - return streamSandboxLeaf({ - task, - signal, - harness, - spec, - seam, - output, - driver, - maxIterations, - controller, - loopCtx: seam.loopCtx, - onArtifact: (a) => { - artifact = a - }, - }) - }, - teardown(_grace): Promise<{ destroyed: boolean }> { - // The composed runAgentRounds owns its box teardown (finally{allSettled(destroy)}); - // aborting the loop's signal cascades into that barrier. - controller.abort() - return Promise.resolve({ destroyed: true }) - }, - resultArtifact() { - if (!artifact) { - throw new ValidationError('sandboxExecutor: resultArtifact() read before stream drained') - } - return artifact + return attestRuntimeOwnedExecutor( + { + runtime: 'sandbox' as Runtime, + execute(task, signal): AsyncIterable { + return streamSandboxLeaf({ + task, + signal, + harness, + spec, + seam, + output, + driver, + maxIterations, + controller, + loopCtx: seam.loopCtx, + onArtifact: (a) => { + artifact = a + }, + }) + }, + teardown(_grace): Promise<{ destroyed: boolean }> { + // The composed runAgentRounds owns its box teardown (finally{allSettled(destroy)}); + // aborting the loop's signal cascades into that barrier. + controller.abort() + return Promise.resolve({ destroyed: true }) + }, + resultArtifact() { + if (!artifact) { + throw new ValidationError('sandboxExecutor: resultArtifact() read before stream drained') + } + return artifact + }, }, - } + sandboxMaterialization, + sandboxBinding, + ) } interface SandboxLeafOut { @@ -772,9 +884,8 @@ async function* streamSandboxLeaf(args: StreamSandboxArgs): AsyncIterable = (_spec, ctx) => { @@ -790,36 +901,64 @@ export const cliExecutor: ExecutorFactory = (_spec, ctx) => { let proc: ReturnType | undefined let artifact: ExecutorResult | undefined + const executionId = ctx.node?.nodeId ?? `cli-process-${randomUUID()}` + const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(executionId) - return { - runtime: 'cli' as Runtime, - budgetExempt: true, - execute(task, signal): AsyncIterable { - return streamCliLeaf({ - task, - signal, - seam, - controller, - onProc: (p) => { - proc = p - }, - onArtifact: (a) => { - artifact = a - }, - }) + return attestRuntimeOwnedExecutor( + { + runtime: 'cli' as Runtime, + budgetExempt: true, + execute(task, signal): AsyncIterable { + return streamCliLeaf({ + task, + signal, + seam, + controller, + onProc: (p) => { + proc = p + }, + onArtifact: (a) => { + artifact = a + }, + }) + }, + async teardown(grace): Promise<{ destroyed: boolean }> { + controller.abort() + if (!proc || proc.exitCode !== null || proc.killed) return { destroyed: true } + return killWithGrace(proc, grace) + }, + resultArtifact() { + if (!artifact) { + throw new ValidationError('cliExecutor: resultArtifact() read before stream drained') + } + return artifact + }, }, - async teardown(grace): Promise<{ destroyed: boolean }> { - controller.abort() - if (!proc || proc.exitCode !== null || proc.killed) return { destroyed: true } - return killWithGrace(proc, grace) + { + effectiveProfile: _spec.profile, + backend: 'cli', + model: { status: 'unknown', reason: 'raw subprocess has no model identity contract' }, + execution: { kind: 'process-attempt', id: executionId }, + materializer: 'raw-cli-stdin', + plan: { + kind: 'raw-cli-process', + bin: seam.bin, + args: seam.args ?? [], + cwd: seam.cwd ?? null, + envOverrides: seam.env ?? {}, + ambientEnvironment: 'inherited', + }, }, - resultArtifact() { - if (!artifact) { - throw new ValidationError('cliExecutor: resultArtifact() read before stream drained') - } - return artifact + { + attemptId, + binding: { + executionId, + bin: seam.bin, + cwd: seam.cwd ?? null, + }, + descriptor: { kind: 'cli-process', transport: 'process', backend: 'cli' }, }, - } + ) } interface StreamCliArgs { @@ -924,26 +1063,29 @@ function killWithGrace( * - STEERABLE: the down-leg `inbox` is drained at each turn boundary; a queued * steer becomes the next turn's prompt on the same session, and the worker can't * settle while a steer it never read is pending (the sandbox/router contract). - * - ABORT: the caller signal + teardown fold into the per-turn fetch signal; a - * forceful (`interrupt`) steer aborts the in-flight turn so the worker re-plans. + * - ABORT: reader abort only detaches HTTP. Interrupt/teardown then call the + * bridge's explicit cancel operation and wait for the owned run to terminate. * * Reports REAL usage when the bridge surfaces it, never a fabricated cost. */ -/** Resolve the bridge wire model for this spawn: a per-create `backend` override - * (harness + model) wins over the seam default, encoded as `${harness}/${model}`. - * Absent an override the seam `model` is used verbatim. */ -function bridgeCellModel(seamModel: string, ctx: ExecutorContext): string { +/** Resolve the bridge wire model for this spawn. Per-create matrix settings win, then the + * canonical profile's harness/model preferences, then the bridge's configured fallback. */ +function bridgeCellModel( + seamModel: string | undefined, + ctx: ExecutorContext, + profile: AgentProfile, +): string | undefined { const create = ctx.seams.createOptions as | { backend?: { type?: string; model?: { model?: string } } } | undefined const backend = create?.backend - const harness = backend?.type - const model = backend?.model?.model + const profileHarness = profile.harness === 'cli-base' ? undefined : profile.harness + const harness = backend?.type ?? profileHarness + const model = backend?.model?.model ?? profile.model?.default if (!harness && !model) return seamModel - const h = harness ?? '' - const m = model ?? seamModel - if (!h) return m - return m.startsWith(`${h}/`) ? m : `${h}/${m}` + if (!harness) return model + if (model) return model.startsWith(`${harness}/`) ? model : `${harness}/${model}` + return seamModel?.startsWith(`${harness}/`) ? seamModel : undefined } export const bridgeExecutor: ExecutorFactory = (spec, ctx) => { @@ -954,16 +1096,20 @@ export const bridgeExecutor: ExecutorFactory = (spec, ctx) => { // the wire id is `${harness}/${model}` (an already-`${harness}/`-prefixed model // passes through). This is how ONE bridge `SandboxClient` drives every // harness×model cell of a matrix — the seam `model` is the fixed default. - const seam = { ...base, model: bridgeCellModel(base.model, ctx) } + const effectiveProfile = agentProfileSchema.parse( + mergeAgentProfiles(spec.profile, base.agentProfile) ?? spec.profile, + ) + const seam = { ...base, model: bridgeCellModel(base.model, ctx, effectiveProfile) } if (!seam.bridgeUrl || !seam.bridgeBearer || !seam.model) { throw new ValidationError( - 'bridgeExecutor: BridgeSeam.bridgeUrl + bridgeBearer + model required', + 'bridgeExecutor: bridgeUrl + bridgeBearer and a profile or bridge model are required', ) } const maxTurns = seam.maxTurns ?? 200 // A stable per-spawn session id (caller can pin one) — cli-bridge keys harness // resume off this exactly as a box id keys a sandbox session. const sessionId = seam.sessionId ?? `bridge-${spec.profile.name ?? 'worker'}-${randomUUID()}` + const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(sessionId) const controller = new AbortController() const abortIfSignalled = () => { @@ -975,50 +1121,97 @@ export const bridgeExecutor: ExecutorFactory = (spec, ctx) => { // The down-leg receive end: the driver's steer/answer/resume land here via `Scope.send`. const inbox = createInbox() let artifact: ExecutorResult | undefined - - return { - runtime: 'cli' as Runtime, - deliver: (m) => inbox.deliver(m), - execute(task, signal): AsyncIterable { - return streamBridgeSession({ - task, - signal, - spec, - seam, - sessionId, - maxTurns, - inbox, - controller, - onArtifact: (a) => { - artifact = a - }, - }) + // One spawn owns one resumable session and, at most, one live durable run per + // turn. Keep the server-owned run identities until the bridge proves each job + // terminal; closing a response body is deliberately not terminal evidence. + const activeRuns = new Map() + + return attestRuntimeOwnedExecutor( + { + runtime: 'cli' as Runtime, + deliver: (m) => inbox.deliver(m), + execute(task, signal): AsyncIterable { + return streamBridgeSession({ + task, + signal, + profile: effectiveProfile, + seam, + sessionId, + maxTurns, + inbox, + controller, + activeRuns, + onArtifact: (a) => { + artifact = a + }, + }) + }, + async teardown(grace): Promise<{ destroyed: boolean }> { + controller.abort() + const remaining = [...activeRuns.values()].filter((run) => !run.terminal) + if (remaining.length === 0) return { destroyed: true } + const terminal = await Promise.all( + remaining.map((run) => cancelBridgeRunToTerminal(seam, run, grace)), + ) + return { destroyed: terminal.every(Boolean) } + }, + resultArtifact() { + if (!artifact) { + throw new ValidationError('bridgeExecutor: resultArtifact() read before stream drained') + } + return { ...artifact, spent: artifact.spent } + }, }, - teardown(_grace): Promise<{ destroyed: boolean }> { - controller.abort() - return Promise.resolve({ destroyed: true }) + { + effectiveProfile, + backend: 'bridge', + model: { status: 'known', id: seam.model }, + execution: { kind: 'session', id: sessionId }, + materializer: 'cli-bridge-agent-profile', + plan: { + kind: 'cli-bridge-session', + bridgeUrl: seam.bridgeUrl, + cwd: seam.cwd ?? null, + maxTurns, + timeoutMs: seam.timeoutMs ?? null, + streaming: true, + }, }, - resultArtifact() { - if (!artifact) { - throw new ValidationError('bridgeExecutor: resultArtifact() read before stream drained') - } - return { ...artifact, spent: artifact.spent } + { + attemptId, + binding: { + bridgeUrl: seam.bridgeUrl, + cwd: seam.cwd ?? null, + effectiveProfile, + model: seam.model, + sessionId, + }, + descriptor: { kind: 'bridge-session', transport: 'http', backend: 'bridge' }, }, - } + ) } interface StreamBridgeArgs { task: unknown signal: AbortSignal - spec: AgentSpec + profile: AgentProfile seam: BridgeSeam sessionId: string maxTurns: number inbox: Inbox controller: AbortController + activeRuns: Map onArtifact: (a: ExecutorResult) => void } +interface ActiveBridgeRun { + readonly id: string + requestDigest?: string + lastEventId: number + terminal: boolean + cancelInFlight?: Promise +} + /** * One resumable cli-bridge session, run as a streamed turn loop. Turn 0 sends the * task; each subsequent turn fires ONLY when the inbox has a steer/answer to fold — @@ -1031,7 +1224,9 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable 0 they ARE the prompt (resume content). const pending = inbox.drain() @@ -1050,13 +1243,10 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable = [] - if (t === 0 && typeof system === 'string' && system.length > 0) { - messages.push({ role: 'system', content: system }) - } messages.push({ role: 'user', content: nextPrompt }) nextPrompt = undefined @@ -1067,74 +1257,103 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable { + timedOut = true + abortTurn() + }, seam.timeoutMs) + : undefined const cleanup = () => { external.removeEventListener('abort', abortTurn) if (timer) clearTimeout(timer) } - let res: BridgeResponse - try { - res = await bridgeStreamPost(seam.bridgeUrl, { - bearer: seam.bridgeBearer, - sessionId: args.sessionId, - body: { - model: seam.model, - stream: true, - session_id: args.sessionId, - ...(seam.cwd ? { cwd: seam.cwd } : {}), - ...(seam.agentProfile ? { agent_profile: seam.agentProfile } : {}), - messages, - }, - signal: turnController.signal, - }) - } catch (e) { - cleanup() - // Re-plan ONLY when a forceful steer (not external teardown) aborted the turn — - // the steer is already queued, so loop back and fold it. Anything else is fatal. - const interruptAbort = - e instanceof DOMException && - e.name === 'AbortError' && - interruptSig.aborted && - !args.signal.aborted && - !args.controller.signal.aborted - if (interruptAbort) continue - throw e - } - if (!res.ok) { - cleanup() - throw new ValidationError( - `bridgeExecutor: bridge ${res.status}: ${(await res.text()).slice(0, 300)}`, - ) + const activeRun: ActiveBridgeRun = { + id: `bridge-run-${randomUUID()}`, + lastEventId: 0, + terminal: false, } - if (!res.body) { - cleanup() - throw new ValidationError('bridgeExecutor: bridge response had no body to stream') + args.activeRuns.set(activeRun.id, activeRun) + const requestBody = { + model: seam.model, + stream: true, + run_id: activeRun.id, + session_id: args.sessionId, + ...(seam.cwd ? { cwd: seam.cwd } : {}), + agent_profile: args.profile, + messages, } let turnText = '' + let turnTokensKnown = false + let turnUsdKnown = false + let interrupted = false try { - for await (const chunk of parseSseChatStream(res.body)) { + for await (const chunk of streamDurableBridgeRun({ + seam, + sessionId: args.sessionId, + body: requestBody, + signal: turnController.signal, + run: activeRun, + })) { if (chunk.content) { turnText += chunk.content } if (chunk.toolCall) toolCalls.push(chunk.toolCall) if (chunk.usage) { + turnTokensKnown = true tokens.input += chunk.usage.input tokens.output += chunk.usage.output yield { kind: 'tokens', input: chunk.usage.input, output: chunk.usage.output } } - if (typeof chunk.cost === 'number' && chunk.cost > 0) { - usd += chunk.cost - yield { kind: 'cost', usd: chunk.cost } + if (typeof chunk.cost === 'number') { + turnUsdKnown = true + if (chunk.cost > 0) { + usd += chunk.cost + yield { kind: 'cost', usd: chunk.cost } + } + } + } + } catch (error) { + // A forceful steer first detaches this HTTP reader, then explicitly cancels + // the durable run and waits for terminal proof. Starting the resume turn + // before that acknowledgement would race two harness processes against one + // resumable session. + const interruptAbort = + interruptSig.aborted && !args.signal.aborted && !args.controller.signal.aborted + if (interruptAbort) { + const terminal = await cancelBridgeRunToTerminal(seam, activeRun, 'infinity', external) + if (!terminal) { + throw new ValidationError( + `bridgeExecutor: interrupted run ${activeRun.id} did not reach terminal state`, + ) + } + interrupted = true + } else { + // A per-turn timeout is owned here, not by the HTTP socket. Request + // explicit cancellation before surfacing it; external scope teardown + // performs the same operation under its own grace budget. + if (timedOut && !activeRun.terminal) { + await requestBridgeRunCancellation(seam, activeRun, 0) } + throw error } } finally { cleanup() } + // Some transports can finish a buffered body normally after their signal fires. The forceful + // steer still wins and must become a new turn rather than letting this response settle. + if (interruptSig.aborted && !args.signal.aborted && !args.controller.signal.aborted) { + interrupted = true + } turns += 1 + if (!turnTokensKnown) tokensKnown = false + if (!turnUsdKnown) usdKnown = false yield { kind: 'iteration' } - if (turnText) lastText = turnText + if (!interrupted && turnText) lastText = turnText + + if (interrupted) continue // Before settling, drain once more — the worker can't finish while a steer it // never read is pending (the sandbox/router settle contract). A pending steer @@ -1145,7 +1364,9 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable { + let reconnects = 0 + let pendingUpstreamError: ValidationError | undefined + + for (;;) { + let res: BridgeResponse + try { + res = await bridgeStreamPost(args.seam.bridgeUrl, { + bearer: args.seam.bridgeBearer, + sessionId: args.sessionId, + runId: args.run.id, + afterEventId: args.run.lastEventId, + body: args.body, + signal: args.signal, + }) + } catch (error) { + if (args.signal.aborted) throw error + if (reconnects >= BRIDGE_MAX_RECONNECTS) { + throw new ValidationError( + `bridgeExecutor: run ${args.run.id} disconnected before terminal acknowledgement after ${reconnects + 1} attempts: ${errorMessage(error)}`, + ) + } + reconnects += 1 + continue + } + + if (!res.ok) { + throw new ValidationError( + `bridgeExecutor: bridge ${res.status}: ${(await res.text()).slice(0, 300)}`, + ) + } + if (!res.body) { + throw new ValidationError('bridgeExecutor: bridge response had no body to stream') + } + assertBridgeResponseIdentity(res, args.run) + + let sawDone = false + try { + for await (const event of parseSseChatStream(res.body)) { + if (event.kind === 'done') { + sawDone = true + break + } + const expected = args.run.lastEventId + 1 + if (event.id !== expected) { + throw new ValidationError( + `bridgeExecutor: run ${args.run.id} replay gap: expected event ${expected}, received ${event.id}`, + ) + } + args.run.lastEventId = event.id + if (event.error) pendingUpstreamError = event.error + if (event.chunk) yield event.chunk + } + } catch (error) { + if (args.signal.aborted) throw error + if (error instanceof ValidationError) throw error + if (reconnects >= BRIDGE_MAX_RECONNECTS) { + throw new ValidationError( + `bridgeExecutor: run ${args.run.id} stream disconnected before terminal acknowledgement after ${reconnects + 1} attempts: ${errorMessage(error)}`, + ) + } + reconnects += 1 + continue + } + + if (sawDone) { + args.run.terminal = true + if (pendingUpstreamError) throw pendingUpstreamError + return + } + // Preserve the provider's actual diagnostic even when a non-conforming + // bridge drops the final [DONE]. The run remains nonterminal in our local + // state, so teardown still has to cancel and obtain real terminal proof. + if (pendingUpstreamError) throw pendingUpstreamError + if (args.signal.aborted) { + throw new DOMException('bridgeExecutor: turn aborted', 'AbortError') + } + if (reconnects >= BRIDGE_MAX_RECONNECTS) { + throw new ValidationError( + `bridgeExecutor: run ${args.run.id} ended without terminal acknowledgement after ${reconnects + 1} attempts`, + ) + } + reconnects += 1 + } +} + /** The subset of `Response` `streamBridgeSession` consumes: status gate, an error * body reader, and a web `ReadableStream` the SSE parser drains. */ interface BridgeResponse { ok: boolean status: number + headers: Readonly> text: () => Promise body: ReadableStream | null } @@ -1168,6 +1496,8 @@ interface BridgeResponse { interface BridgeStreamPostArgs { bearer: string sessionId: string + runId: string + afterEventId: number body: unknown signal: AbortSignal } @@ -1201,6 +1531,8 @@ function bridgeStreamPost(url: string, args: BridgeStreamPostArgs): Promise 0 ? { 'last-event-id': String(args.afterEventId) } : {}), 'content-length': Buffer.byteLength(payload), }, // No header/body idle timeout: a slow bridge is a live bridge; the abort @@ -1208,12 +1540,15 @@ function bridgeStreamPost(url: string, args: BridgeStreamPostArgs): Promise { + response = res + res.once('close', () => args.signal.removeEventListener('abort', onAbort)) const status = res.statusCode ?? 0 const ok = status >= 200 && status < 300 const body = Readable.toWeb(res) as ReadableStream resolve({ ok, status, + headers: res.headers, body, text: async () => { const chunks: Buffer[] = [] @@ -1223,8 +1558,12 @@ function bridgeStreamPost(url: string, args: BridgeStreamPostArgs): Promise[0] | undefined const onAbort = (): void => { req.destroy(new DOMException('bridgeExecutor: turn aborted', 'AbortError')) + if (response && 'destroy' in response && typeof response.destroy === 'function') { + response.destroy(new DOMException('bridgeExecutor: turn aborted', 'AbortError')) + } } if (args.signal.aborted) onAbort() else args.signal.addEventListener('abort', onAbort, { once: true }) @@ -1232,12 +1571,187 @@ function bridgeStreamPost(url: string, args: BridgeStreamPostArgs): Promise args.signal.removeEventListener('abort', onAbort)) + req.on('close', () => { + if (!response) args.signal.removeEventListener('abort', onAbort) + }) req.write(payload) req.end() }) } +interface BridgeBufferedResponse { + status: number + headers: Readonly> + text: string +} + +function bridgeHeader( + headers: Readonly>, + name: string, +): string | undefined { + const raw = headers[name.toLowerCase()] + if (Array.isArray(raw)) return raw.length === 1 ? raw[0] : undefined + return raw +} + +function assertBridgeResponseIdentity(response: BridgeResponse, run: ActiveBridgeRun): void { + assertBridgeIdentityHeaders(response.headers, run) +} + +function assertBridgeIdentityHeaders( + headers: Readonly>, + run: ActiveBridgeRun, +): void { + const responseRunId = bridgeHeader(headers, 'x-run-id') + if (responseRunId !== run.id) { + throw new ValidationError( + `bridgeExecutor: bridge run identity mismatch: expected ${run.id}, received ${responseRunId ?? 'missing'}`, + ) + } + const digest = bridgeHeader(headers, 'x-run-request-digest') + if (!digest || !/^sha256:[a-f0-9]{64}$/u.test(digest)) { + throw new ValidationError('bridgeExecutor: bridge response omitted a valid request digest') + } + if (run.requestDigest !== undefined && run.requestDigest !== digest) { + throw new ValidationError( + `bridgeExecutor: bridge request digest changed for run ${run.id}: expected ${run.requestDigest}, received ${digest}`, + ) + } + run.requestDigest = digest +} + +/** Explicitly cancel one server-owned run and long-poll for its terminal snapshot. */ +function bridgeCancelPost( + seam: BridgeSeam, + run: ActiveBridgeRun, + waitMs: number, +): Promise { + const target = new URL( + `${seam.bridgeUrl.replace(/\/$/, '')}/v1/runs/${encodeURIComponent(run.id)}/cancel`, + ) + target.searchParams.set('wait_ms', String(waitMs)) + const requestFn = target.protocol === 'https:' ? httpsRequest : httpRequest + return new Promise((resolve, reject) => { + const req = requestFn( + target, + { + method: 'POST', + headers: { + authorization: `Bearer ${seam.bridgeBearer}`, + 'x-run-id': run.id, + 'content-length': '0', + }, + timeout: 0, + }, + (res) => { + void (async () => { + const chunks: Buffer[] = [] + for await (const chunk of res) chunks.push(Buffer.from(chunk)) + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + text: Buffer.concat(chunks).toString('utf8'), + }) + })().catch(reject) + }, + ) + req.on('error', reject) + req.end() + }) +} + +async function requestBridgeRunCancellation( + seam: BridgeSeam, + run: ActiveBridgeRun, + waitMs: number, +): Promise { + if (run.terminal) return true + if (run.cancelInFlight) return run.cancelInFlight + const work = (async (): Promise => { + const response = await bridgeCancelPost(seam, run, waitMs) + if (response.status === 404) { + throw new ValidationError( + `bridgeExecutor: bridge no longer knows run ${run.id}; terminal state is unproven`, + ) + } + if (response.status !== 200 && response.status !== 202) { + throw new ValidationError( + `bridgeExecutor: cancel ${run.id} returned ${response.status}: ${response.text.slice(0, 300)}`, + ) + } + assertBridgeIdentityHeaders(response.headers, run) + let parsed: { + terminal?: unknown + run?: { id?: unknown; requestDigest?: unknown; terminal?: unknown } + } + try { + parsed = JSON.parse(response.text) as typeof parsed + } catch { + throw new ValidationError(`bridgeExecutor: cancel ${run.id} returned invalid JSON`) + } + if ( + parsed.run?.id !== run.id || + parsed.run.requestDigest !== run.requestDigest || + typeof parsed.terminal !== 'boolean' || + typeof parsed.run.terminal !== 'boolean' || + parsed.terminal !== parsed.run.terminal + ) { + throw new ValidationError( + `bridgeExecutor: cancel ${run.id} returned an inconsistent terminal snapshot`, + ) + } + if (response.status === 200 && parsed.terminal === true) { + run.terminal = true + return true + } + if (response.status === 202 && parsed.terminal === false) return false + throw new ValidationError( + `bridgeExecutor: cancel ${run.id} status ${response.status} disagreed with terminal=${String(parsed.terminal)}`, + ) + })() + run.cancelInFlight = work + try { + return await work + } finally { + if (run.cancelInFlight === work) run.cancelInFlight = undefined + } +} + +async function cancelBridgeRunToTerminal( + seam: BridgeSeam, + run: ActiveBridgeRun, + grace: number | 'brutalKill' | 'infinity', + stopSignal?: AbortSignal, +): Promise { + if (run.terminal) return true + const deadline = + grace === 'infinity' + ? undefined + : Date.now() + (grace === 'brutalKill' ? BRIDGE_BRUTAL_KILL_WAIT_MS : Math.max(0, grace)) + let first = true + for (;;) { + const remaining = deadline === undefined ? BRIDGE_CANCEL_LONG_POLL_MS : deadline - Date.now() + if (!first && remaining <= 0) return false + if (!first && stopSignal?.aborted) return false + const waitMs = Math.max( + 0, + Math.min( + stopSignal ? 1_000 : BRIDGE_CANCEL_LONG_POLL_MS, + deadline === undefined ? remaining : Math.max(0, remaining), + ), + ) + const terminal = await requestBridgeRunCancellation(seam, run, waitMs) + if (terminal) return true + first = false + if (deadline !== undefined && Date.now() >= deadline) return false + await new Promise((resolve) => setTimeout(resolve, 10)) + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + interface BridgeStreamChunk { content?: string toolCall?: string @@ -1245,15 +1759,19 @@ interface BridgeStreamChunk { cost?: number } +type BridgeSseEvent = + | { kind: 'event'; id: number; chunk?: BridgeStreamChunk; error?: ValidationError } + | { kind: 'done' } + /** * Parse cli-bridge's OpenAI-compatible SSE stream into normalized chunks. Each - * `data:` line is an OpenAI chat-completion chunk (`choices[].delta`); `[DONE]` - * and SSE comments (`:` keepalives) terminate/skip. Mirrors how `streamSandboxLeaf` - * folds a box's event stream — same `UsageEvent` currency, different wire shape. + * `data:` line is an OpenAI chat-completion chunk (`choices[].delta`). Every + * run-owned frame, including an id-only comment, is returned so the caller can + * prove a contiguous replay sequence. Transport keepalives have no id and are ignored. */ async function* parseSseChatStream( body: ReadableStream, -): AsyncIterable { +): AsyncIterable { const reader = body.getReader() const decoder = new TextDecoder() let buf = '' @@ -1263,16 +1781,16 @@ async function* parseSseChatStream( if (done) break buf += decoder.decode(value, { stream: true }) // SSE frames are separated by a blank line; split on it and keep the tail. - let sep = buf.indexOf('\n\n') - while (sep !== -1) { - const frame = buf.slice(0, sep) - buf = buf.slice(sep + 2) - const chunk = parseSseFrame(frame) - if (chunk === 'done') return - if (chunk) yield chunk - sep = buf.indexOf('\n\n') + let separator = /\r?\n\r?\n/u.exec(buf) + while (separator) { + const frame = buf.slice(0, separator.index) + buf = buf.slice(separator.index + separator[0].length) + const event = parseSseFrame(frame) + if (event) yield event + separator = /\r?\n\r?\n/u.exec(buf) } } + buf += decoder.decode() // Upstream failures routinely arrive UNTERMINATED: a final `data:` frame // with no trailing blank line, or a bare JSON error body with no SSE // framing at all (kimi's access_terminated_error). Dropping the tail here @@ -1280,7 +1798,7 @@ async function* parseSseChatStream( // fails the run, but the diagnostic dies with the buffer. Parse the tail so // the upstream error message rides the thrown event instead. const tail = parseSseStreamTail(buf) - if (tail !== undefined && tail !== 'done') yield tail + if (tail !== undefined) yield tail } finally { reader.releaseLock() } @@ -1290,7 +1808,7 @@ async function* parseSseChatStream( * blank line, or a bare (non-SSE) JSON body — the shape bridge upstreams use * for terminal failures. Throws `ValidationError` on an error payload; returns * `undefined` for keepalive noise or non-JSON leftovers. */ -function parseSseStreamTail(buf: string): BridgeStreamChunk | 'done' | undefined { +function parseSseStreamTail(buf: string): BridgeSseEvent | undefined { const tail = buf.trim() if (!tail) return undefined const framed = parseSseFrame(tail) @@ -1309,18 +1827,32 @@ function parseSseStreamTail(buf: string): BridgeStreamChunk | 'done' | undefined return undefined } -/** Parse one SSE frame (possibly multi-line `data:`/comment) into a chunk, `'done'`, - * or undefined (comment/keepalive/empty). */ -function parseSseFrame(frame: string): BridgeStreamChunk | 'done' | undefined { +/** Parse one SSE frame into a numbered run event, terminal marker, or unnumbered keepalive. */ +function parseSseFrame(frame: string): BridgeSseEvent | undefined { const dataLines: string[] = [] + let id: number | undefined for (const rawLine of frame.split('\n')) { const line = rawLine.replace(/\r$/, '') - if (!line || line.startsWith(':')) continue // comment / keepalive + if (!line || line.startsWith(':')) continue + if (line.startsWith('id:')) { + const rawId = line.slice('id:'.length).trim() + if (!/^[1-9][0-9]*$/u.test(rawId)) { + throw new ValidationError(`bridgeExecutor: invalid SSE event id ${JSON.stringify(rawId)}`) + } + const parsedId = Number(rawId) + if (!Number.isSafeInteger(parsedId)) { + throw new ValidationError(`bridgeExecutor: SSE event id exceeds safe integer range`) + } + id = parsedId + continue + } if (line.startsWith('data:')) dataLines.push(line.slice('data:'.length).trimStart()) } - if (dataLines.length === 0) return undefined + if (dataLines.length === 0) { + return id === undefined ? undefined : { kind: 'event', id } + } const data = dataLines.join('\n') - if (data === '[DONE]') return 'done' + if (data === '[DONE]') return { kind: 'done' } let parsed: { choices?: Array<{ delta?: { @@ -1335,14 +1867,21 @@ function parseSseFrame(frame: string): BridgeStreamChunk | 'done' | undefined { try { parsed = JSON.parse(data) } catch { - return undefined + throw new ValidationError('bridgeExecutor: bridge emitted a non-JSON SSE data frame') + } + if (id === undefined) { + throw new ValidationError('bridgeExecutor: bridge emitted an unnumbered run event') } if (parsed.error) { // `type` is the upstream's error class (e.g. kimi's access_terminated_error) // — carry it when the payload has no message, never collapse to 'unknown'. - throw new ValidationError( - `bridgeExecutor: bridge stream error: ${parsed.error.message ?? parsed.error.type ?? 'unknown'}`, - ) + return { + kind: 'event', + id, + error: new ValidationError( + `bridgeExecutor: bridge stream error: ${parsed.error.message ?? parsed.error.type ?? 'unknown'}`, + ), + } } const out: BridgeStreamChunk = {} const choice = parsed.choices?.[0] @@ -1355,7 +1894,11 @@ function parseSseFrame(frame: string): BridgeStreamChunk | 'done' | undefined { out.usage = { input: u.prompt_tokens ?? 0, output: u.completion_tokens ?? 0 } } if (typeof u?.cost === 'number') out.cost = u.cost - return Object.keys(out).length > 0 ? out : undefined + return { + kind: 'event', + id, + ...(Object.keys(out).length > 0 ? { chunk: out } : {}), + } } function bridgeWorktreeExecutor( @@ -1375,6 +1918,11 @@ function bridgeWorktreeExecutor( const runId = seam.runId ?? randomUUID() const sessionId = bridge.sessionId ?? `bridge-worktree-${runId}` + const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(runId) + const effectiveProfile = agentProfileSchema.parse( + mergeAgentProfiles(spec.profile, bridge.agentProfile) ?? spec.profile, + ) + const model = bridgeCellModel(bridge.model, ctx, effectiveProfile) const controller = new AbortController() const pending: unknown[] = [] let inner: Executor | undefined @@ -1402,129 +1950,158 @@ function bridgeWorktreeExecutor( pending.push(msg) } - return { - runtime: 'cli' as Runtime, - budgetExempt: seam.budgetExempt ?? false, - deliver, - execute(_task, signal): AsyncIterable { - return (async function* bridgeWorktreeStream() { - const started = Date.now() - const linked = mergeAbortSignals(signal, controller.signal) - let bridgeArtifact: ExecutorResult | undefined - - try { - worktree = await createWorktree({ - repoRoot: seam.repoRoot, - runId, - ...(seam.baseRef ? { baseRef: seam.baseRef } : {}), - ...(seam.runGit ? { runGit: seam.runGit } : {}), - }) - removed = false - - const bridgeSeam: BridgeSeam = { - bridgeUrl: bridge.bridgeUrl, - bridgeBearer: bridge.bridgeBearer, - model: resolveBridgeWorktreeModel(spec, bridge), - cwd: worktree.path, - sessionId, - ...(bridge.agentProfile - ? { agentProfile: bridge.agentProfile } - : { agentProfile: spec.profile as unknown as Record }), - ...(bridge.timeoutMs !== undefined ? { timeoutMs: bridge.timeoutMs } : {}), - ...(bridge.maxTurns !== undefined ? { maxTurns: bridge.maxTurns } : {}), - } - const bridgeCtx: ExecutorContext = { - ...ctx, - signal: linked, - seams: { ...ctx.seams, [bridgeSeamKey]: bridgeSeam }, - } - inner = bridgeExecutor(spec, bridgeCtx) - for (const msg of pending.splice(0)) inner.deliver?.(msg) - - const run = inner.execute(seam.taskPrompt, linked) - if (isAsyncIterable(run)) { - for await (const event of run) yield event - bridgeArtifact = inner.resultArtifact() - } else { - bridgeArtifact = await run - } + return attestRuntimeOwnedExecutor( + { + runtime: 'cli' as Runtime, + budgetExempt: seam.budgetExempt ?? false, + deliver, + execute(task, signal): AsyncIterable { + return (async function* bridgeWorktreeStream() { + const started = Date.now() + const linked = mergeAbortSignals(signal, controller.signal) + let bridgeArtifact: ExecutorResult | undefined - const diff = await captureWorktreeDiff({ - worktree, - ...(seam.runGit ? { runGit: seam.runGit } : {}), - }) - const checks = await runWorktreeChecks({ - worktreePath: worktree.path, - ...(seam.testCmd !== undefined ? { testCmd: seam.testCmd } : {}), - ...(seam.typecheckCmd !== undefined ? { typecheckCmd: seam.typecheckCmd } : {}), - timeoutMs: - seam.checkTimeoutMs ?? seam.harnessTimeoutMs ?? bridge.timeoutMs ?? 5 * 60 * 1000, - cap: seam.checkOutputCap ?? 16_000, - ...(seam.runCommand ? { runCommand: seam.runCommand } : {}), - signal: linked, - }) + try { + worktree = await createWorktree({ + repoRoot: seam.repoRoot, + runId, + ...(seam.baseRef ? { baseRef: seam.baseRef } : {}), + ...(seam.runGit ? { runGit: seam.runGit } : {}), + }) + removed = false + + const bridgeSeam: BridgeSeam = { + bridgeUrl: bridge.bridgeUrl, + bridgeBearer: bridge.bridgeBearer, + cwd: worktree.path, + sessionId, + ...(bridge.model ? { model: bridge.model } : {}), + ...(bridge.agentProfile ? { agentProfile: bridge.agentProfile } : {}), + ...(bridge.timeoutMs !== undefined ? { timeoutMs: bridge.timeoutMs } : {}), + ...(bridge.maxTurns !== undefined ? { maxTurns: bridge.maxTurns } : {}), + } + const bridgeCtx: ExecutorContext = { + ...ctx, + signal: linked, + seams: { ...ctx.seams, [bridgeSeamKey]: bridgeSeam }, + } + inner = bridgeExecutor(spec, bridgeCtx) + for (const msg of pending.splice(0)) inner.deliver?.(msg) + + const run = inner.execute(task, linked) + if (isAsyncIterable(run)) { + for await (const event of run) yield event + bridgeArtifact = inner.resultArtifact() + } else { + bridgeArtifact = await run + } + + const diff = await captureWorktreeDiff({ + worktree, + ...(seam.runGit ? { runGit: seam.runGit } : {}), + }) + const checks = await runWorktreeChecks({ + worktreePath: worktree.path, + ...(seam.testCmd !== undefined ? { testCmd: seam.testCmd } : {}), + ...(seam.typecheckCmd !== undefined ? { typecheckCmd: seam.typecheckCmd } : {}), + timeoutMs: + seam.checkTimeoutMs ?? seam.harnessTimeoutMs ?? bridge.timeoutMs ?? 5 * 60 * 1000, + cap: seam.checkOutputCap ?? 16_000, + ...(seam.runCommand ? { runCommand: seam.runCommand } : {}), + signal: linked, + }) - const result: WorktreeHarnessResult = { - branch: worktree.branch, - patch: diff.patch, - stats: diff.stats, - harness: { - name: 'bridge', - exitCode: null, - timedOut: false, - killedBySignal: null, - durationMs: bridgeArtifact.spent.ms || Date.now() - started, - stdout: bridgeOutputText(bridgeArtifact.out), - stderr: '', - }, - ...(checks ? { checks } : {}), - } - const spent: Spend = { - ...bridgeArtifact.spent, - ms: bridgeArtifact.spent.ms || Date.now() - started, + const result: WorktreeHarnessResult = { + branch: worktree.branch, + patch: diff.patch, + stats: diff.stats, + harness: { + name: 'bridge', + exitCode: null, + timedOut: false, + killedBySignal: null, + durationMs: bridgeArtifact.spent.ms || Date.now() - started, + stdout: bridgeOutputText(bridgeArtifact.out), + stderr: '', + }, + ...(checks ? { checks } : {}), + } + const spent: Spend = { + ...bridgeArtifact.spent, + ms: bridgeArtifact.spent.ms || Date.now() - started, + } + artifact = { + outRef: contentRef('bridge-worktree', { sessionId, result }), + out: result, + spent, + } + } catch (err) { + controller.abort() + await inner?.teardown('brutalKill').catch(() => undefined) + await cleanupWorktree() + throw err } - artifact = { - outRef: contentRef('bridge-worktree', { sessionId, result }), - out: result, - spent, + })() + }, + async teardown(grace): Promise<{ destroyed: boolean }> { + controller.abort() + let destroyed = true + try { + if (inner) { + destroyed = (await inner.teardown(grace)).destroyed } - } catch (err) { - controller.abort() - await inner?.teardown('brutalKill').catch(() => undefined) + } finally { await cleanupWorktree() - throw err } - })() - }, - async teardown(grace): Promise<{ destroyed: boolean }> { - controller.abort() - let destroyed = true - try { - if (inner) { - destroyed = (await inner.teardown(grace)).destroyed + return { destroyed } + }, + resultArtifact() { + if (!artifact) { + throw new ValidationError( + 'cliWorktreeExecutor: bridge resultArtifact() read before stream drained', + ) } - } finally { - await cleanupWorktree() - } - return { destroyed } + return artifact + }, }, - resultArtifact() { - if (!artifact) { - throw new ValidationError( - 'cliWorktreeExecutor: bridge resultArtifact() read before stream drained', - ) - } - return artifact + { + effectiveProfile, + backend: 'bridge-worktree', + model: model + ? { status: 'known', id: model } + : { status: 'unknown', reason: 'bridge worktree profile did not select a model' }, + execution: { kind: 'worktree-session', id: `${runId}:${sessionId}` }, + materializer: 'bridge-worktree-agent-profile', + plan: { + kind: 'bridge-worktree-session', + runId, + sessionId, + baseRef: seam.baseRef ?? 'HEAD', + bridgeUrl: bridge.bridgeUrl, + model: model ?? null, + testCmd: seam.testCmd ?? null, + typecheckCmd: seam.typecheckCmd ?? null, + checkTimeoutMs: + seam.checkTimeoutMs ?? seam.harnessTimeoutMs ?? bridge.timeoutMs ?? 5 * 60 * 1000, + checkOutputCap: seam.checkOutputCap ?? 16_000, + }, + }, + { + attemptId, + binding: { + bridgeUrl: bridge.bridgeUrl, + effectiveProfile, + model: model ?? null, + repoRoot: seam.repoRoot, + runId, + sessionId, + }, + descriptor: { + kind: 'bridge-worktree-session', + transport: 'http', + backend: 'bridge-worktree', + }, }, - } -} - -function resolveBridgeWorktreeModel(spec: AgentSpec, bridge: CliWorktreeBridgeSeam): string { - if (bridge.model) return bridge.model - const model = spec.profile.model?.default - if (typeof model === 'string' && model.length > 0) return model - throw new ValidationError( - 'cliWorktreeExecutor: bridge.model or AgentProfile.model.default required', ) } @@ -1558,8 +2135,8 @@ function isAsyncIterable(value: unknown): value is AsyncIterable { */ export const cliWorktreeExecutor: ExecutorFactory = (spec, ctx) => { const seam = readSeam(ctx, cliWorktreeSeamKey, 'cli-worktree') - if (!seam.repoRoot || !seam.taskPrompt) { - throw new ValidationError('cliWorktreeExecutor: CliWorktreeSeam.repoRoot + taskPrompt required') + if (!seam.repoRoot) { + throw new ValidationError('cliWorktreeExecutor: CliWorktreeSeam.repoRoot required') } if (seam.bridge) return bridgeWorktreeExecutor(spec, ctx, seam) if (!seam.harness) { @@ -1571,7 +2148,7 @@ export const cliWorktreeExecutor: ExecutorFactory = (spec, ctx) => { repoRoot: seam.repoRoot, profile: spec.profile, harness: seam.harness, - taskPrompt: seam.taskPrompt, + ...(seam.taskPrompt !== undefined ? { taskPrompt: seam.taskPrompt } : {}), ...(seam.runId ? { runId: seam.runId } : {}), ...(seam.baseRef ? { baseRef: seam.baseRef } : {}), ...(seam.harnessTimeoutMs !== undefined ? { harnessTimeoutMs: seam.harnessTimeoutMs } : {}), @@ -1584,6 +2161,7 @@ export const cliWorktreeExecutor: ExecutorFactory = (spec, ctx) => { ...(seam.runGit ? { runGit: seam.runGit } : {}), ...(seam.runCommand ? { runCommand: seam.runCommand } : {}), ...(seam.budgetExempt !== undefined ? { budgetExempt: seam.budgetExempt } : {}), + ...(ctx.node?.attemptId !== undefined ? { executionAttemptId: ctx.node.attemptId } : {}), }) as Executor } @@ -1604,6 +2182,139 @@ export type ExecutorConfig = | ({ backend: 'pi' } & PiSeam) | ({ backend: 'sandbox'; harness?: BackendType } & SandboxSeam) +/** Capture one public executor configuration at its call boundary. All data that selects policy, + * model, process, limits, profile overlays, or backend behavior is detached and deeply frozen. + * Explicit service/function fields remain live by reference because they are executable ports, + * not portable configuration. */ +export function snapshotExecutorConfig(config: ExecutorConfig): ExecutorConfig { + switch (config.backend) { + case 'router-tools': { + const { executeToolCall, onToolStep, ...decisionData } = config + const snapshot = detachedSnapshot(decisionData, 'createExecutor router-tools config') + return Object.freeze({ + ...snapshot, + executeToolCall, + ...(onToolStep === undefined ? {} : { onToolStep }), + }) + } + case 'cli-worktree': { + const { runGit, runCommand, ...decisionData } = config + const snapshot = detachedSnapshot(decisionData, 'createExecutor cli-worktree config') + return Object.freeze({ + ...snapshot, + ...(runGit === undefined ? {} : { runGit }), + ...(runCommand === undefined ? {} : { runCommand }), + }) + } + case 'provider': { + const { provider, registry, taskToTurn, ...decisionData } = config + const snapshot = detachedSnapshot(decisionData, 'createExecutor provider config') + // A registry is a live service. Resolve its mutable name mapping exactly once at intake and + // retain the resulting provider instance, never the registry lookup for later execution. + const resolvedProvider = resolveAgentEnvironmentProvider(provider, registry) + return Object.freeze({ + ...snapshot, + provider: resolvedProvider, + ...(taskToTurn === undefined ? {} : { taskToTurn }), + }) + } + case 'sandbox': { + const { sandboxClient, loopCtx, ...decisionData } = config + if (loopCtx === undefined) { + const snapshot = detachedSnapshot(decisionData, 'createExecutor sandbox config') + return Object.freeze({ ...snapshot, sandboxClient }) + } + const { hooks, traceEmitter, onSandboxEvent, runHandle, ...loopDecisionData } = loopCtx + const snapshot = detachedSnapshot( + { ...decisionData, loopCtx: loopDecisionData }, + 'createExecutor sandbox config', + ) + const loopSnapshot = snapshot.loopCtx + return Object.freeze({ + ...snapshot, + sandboxClient, + loopCtx: Object.freeze({ + ...loopSnapshot, + ...(hooks === undefined ? {} : { hooks }), + ...(traceEmitter === undefined ? {} : { traceEmitter }), + ...(onSandboxEvent === undefined ? {} : { onSandboxEvent }), + ...(runHandle === undefined ? {} : { runHandle }), + }), + }) + } + case 'router': + case 'bridge': + case 'cli': + case 'pi': + return detachedSnapshot(config, `createExecutor ${config.backend} config`) + } +} + +/** A backend config reused for multiple workers/managers cannot pin execution identity or carry a + * profile overlay applied after Scope hashed the authored profile. Direct single-execution + * `createExecutor` calls may still use those fields. */ +export function captureReusableExecutorConfig( + config: ExecutorConfig, + context: string, +): ExecutorConfig { + const captured = snapshotExecutorConfig(config) + const profileOverlay = + captured.backend === 'bridge' + ? captured.agentProfile + : captured.backend === 'cli-worktree' + ? captured.bridge?.agentProfile + : undefined + if (profileOverlay !== undefined) { + throw new ValidationError( + `${context}: backend agentProfile overlays are not allowed because they change the effective profile after spawn identity is fixed`, + ) + } + const fixedIdentity = + captured.backend === 'bridge' && captured.sessionId !== undefined + ? 'sessionId' + : captured.backend === 'cli-worktree' && captured.runId !== undefined + ? 'runId' + : captured.backend === 'cli-worktree' && captured.bridge?.sessionId !== undefined + ? 'bridge.sessionId' + : undefined + if (fixedIdentity !== undefined) { + throw new ValidationError( + `${context}: fixed ${fixedIdentity} is not allowed on a reusable backend; let each execution derive an isolated id`, + ) + } + return captured +} + +/** Bind one already-captured reusable backend to the durable identity of the execution that will + * use it. Stateful bridge backends need an explicit external id: a random default isolates two + * siblings but cannot reconnect a replacement process to the same harness session. Non-stateful + * backends carry no external execution id and are returned unchanged. */ +export function bindReusableExecutorExecutionId( + captured: ExecutorConfig, + executionId: string, +): ExecutorConfig { + if (typeof executionId !== 'string' || executionId.length === 0) { + throw new ValidationError( + 'bindReusableExecutorExecutionId: executionId must be a non-empty string', + ) + } + switch (captured.backend) { + case 'bridge': + return Object.freeze({ ...captured, sessionId: executionId }) + case 'cli-worktree': + // The bridged worktree derives `bridge-worktree-${runId}` when no inner session id is set, + // so this one durable value binds both the worktree and its resumed harness conversation. + return Object.freeze({ ...captured, runId: executionId }) + case 'router': + case 'router-tools': + case 'cli': + case 'provider': + case 'pi': + case 'sandbox': + return captured + } +} + /** * The single built-in executor factory. Picks a leaf backend by data (`config.backend`), * injects the matching seam, and delegates to that backend's built-in implementation. @@ -1613,10 +2324,12 @@ export type ExecutorConfig = * `UsageEvent` reporting channel. */ export function createExecutor(config: ExecutorConfig): ExecutorFactory { + const captured = snapshotExecutorConfig(config) return (spec, ctx) => { - const { backend, ...seam } = config as ExecutorConfig & Record + const { backend, ...seamData } = captured as ExecutorConfig & Record + const seam = Object.freeze(seamData) const seamed: ExecutorContext = { ...ctx, seams: { ...ctx.seams, [backend]: seam } } - switch (config.backend) { + switch (captured.backend) { case 'router': return routerInlineExecutor(spec, seamed) case 'router-tools': @@ -1640,7 +2353,7 @@ export function createExecutor(config: ExecutorConfig): ExecutorFactory case 'sandbox': { // The sandbox executor requires a concrete harness; a spec-level harness // wins, else the config names it (fail-loud inside if both are absent). - const harness = spec.harness ?? config.harness ?? null + const harness = spec.harness ?? captured.harness ?? null return sandboxExecutor({ ...spec, harness }, seamed) } } @@ -1656,8 +2369,8 @@ export function createExecutor(config: ExecutorConfig): ExecutorFactory * without touching the registry at all. NOT a closed switch; registration + BYO * ARE the extension points. * - * `resolve` precedence (frozen in `ExecutorRegistry`): a BYO `spec.executor` → - * `harness === null` → the `'router'` factory; else a registered factory for the + * `resolve` precedence (frozen in `ExecutorRegistry`): a BYO `spec.executorFactory` → + * `spec.executor` → `harness === null` → the `'router'` factory; else a registered factory for the * harness-derived runtime (`'sandbox'` for any `BackendType`); else fail loud. */ export function createExecutorRegistry(): ExecutorRegistry { @@ -1681,6 +2394,10 @@ export function createExecutorRegistry(): ExecutorRegistry { resolve( spec: AgentSpec, ): { succeeded: true; value: ExecutorFactory } | { succeeded: false; error: string } { + // BYO factory: constructed only after Scope admission with the real signal/context. + if (spec.executorFactory) { + return { succeeded: true, value: spec.executorFactory as ExecutorFactory } + } // BYO: a caller-supplied executor wins, wrapped in a trivial per-spawn factory. if (spec.executor) { const byo = spec.executor @@ -1731,11 +2448,13 @@ function taskToPrompt(task: unknown): string { return JSON.stringify(task) } -/** Router messages from the opaque task + the profile's system prompt, when set. */ +/** Router messages from the opaque task + every portable profile prompt instruction. */ function taskToMessages(task: unknown, spec: AgentSpec): Array<{ role: string; content: string }> { const messages: Array<{ role: string; content: string }> = [] - const system = spec.profile.prompt?.systemPrompt - if (typeof system === 'string' && system.length > 0) { + const system = [spec.profile.prompt?.systemPrompt, ...(spec.profile.prompt?.instructions ?? [])] + .filter((line): line is string => typeof line === 'string' && line.trim().length > 0) + .join('\n') + if (system.length > 0) { messages.push({ role: 'system', content: system }) } messages.push({ role: 'user', content: taskToPrompt(task) }) diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index 902f4a5c..c64c4604 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -25,31 +25,60 @@ * @experimental */ +import { + canonicalCandidateDigest, + type Sha256Digest, + sha256DigestSchema, +} from '@tangle-network/agent-interface' import { contentAddress } from '../../durable/spawn-journal' import { ValidationError } from '../../errors' import { notifyRuntimeHookEvent, type RuntimeHooks } from '../../runtime-hooks' import type { Iteration } from '../types' -import type { BudgetPool, ReservationTicket } from './budget' +import { type BudgetPool, createBudgetPool, type ReservationTicket } from './budget' +import { + armDeadlineTimer, + boundedChildDeadlineAt, + DEFAULT_SUCCESSFUL_SHUTDOWN_MS, + teardownExecutor, +} from './deadline' +import { freeSlots } from './dispatch' +import { + authoredProfileDigest, + knownExecutionBindingReceipt, + knownMaterializationReceipt, + newExecutionAttemptId, + runtimeOwnedDeferredExecutorRuntime, + runtimeOwnedExecutorExecutionBinding, + runtimeOwnedExecutorMaterialization, + unknownExecutionBindingReceipt, + unknownMaterializationReceipt, +} from './materialization' import { DEFAULT_STALL_AFTER_MS, type ExecutorProgress, readWorkerProgress, type WorkerProgress, } from './progress' +import { detachedSnapshot } from './snapshot' import type { TraceSource } from './trace-source' import type { Agent, AgentSpec, Budget, DefaultVerdict, + ExecutionBindingReceipt, Executor, ExecutorContext, + ExecutorExecutionBinding, + ExecutorNodeContext, ExecutorRegistry, ExecutorResult, Handle, + NodeExecutionIdentity, NodeId, NodeSnapshot, NodeStatus, + ProfileMaterializationReceipt, ResultBlobStore, ResumedKeyState, ResumedWork, @@ -83,7 +112,7 @@ export interface ScopeArgs { readonly parentId: NodeId /** Journal/blob root key the supervisor `beginTree`'d. */ readonly root: NodeId - /** The shared conserved reservation pool (one per supervised run). */ + /** The reservation pool for this scope: the root total or one nested allocated partition. */ readonly pool: BudgetPool /** Append-only spawn journal; this scope writes `spawned` + `settled` records. */ readonly journal: SpawnJournal @@ -102,6 +131,11 @@ export interface ScopeArgs { readonly depth: number /** Runtime recursion-depth ceiling — a spawn past it fails closed `depth-exceeded`. */ readonly maxDepth?: number + /** Root-owned limit on live spawned workers across this scope and every nested scope. */ + readonly maxLiveWorkers?: number + /** @internal Shared counter inherited by nested scopes. Callers set `maxLiveWorkers`; the root + * scope creates this state once and passes the same object through its recursion seam. */ + readonly liveWorkerCapacity?: LiveWorkerCapacityState /** Abort signal for this scope; an abort cascades into every live child's executor. */ readonly signal: AbortSignal /** Injected clock — keeps the journal `at` timestamp deterministic in tests. */ @@ -110,6 +144,20 @@ export interface ScopeArgs { * SAME stream `runAgentRounds`/`tool-loop` feed, so the recursive tree is ONE observable stream * (the topology viewer reads it). Undefined ⇒ the journal stays the only record. */ readonly hooks?: RuntimeHooks + /** @internal Trusted root-adapter publication channel. It is never exposed as a Scope method. */ + readonly ownerMaterialization?: { + readonly runtime: NodeSnapshot['runtime'] + readonly authoredProfile?: unknown + readonly attemptId: string + readonly prior?: ProfileMaterializationReceipt + readonly journalRoot?: NodeId + readonly nodeId?: NodeId + readonly requiredKnown?: boolean + readonly onReceipt?: ( + materialization: ProfileMaterializationReceipt, + binding: ExecutionBindingReceipt, + ) => void + } /** * Resume seam — set ONLY by the supervisor when `SupervisorOpts.resume` is on AND a non-empty * journal tree exists for this root. It carries the replayed committed work (so `scope.resume` @@ -135,6 +183,12 @@ export interface ScopeArgs { } } +/** Mutable only inside Scope admission/release. Every nested scope receives this exact object. */ +export interface LiveWorkerCapacityState { + readonly max: number | undefined + live: number +} + /** * Internal live-set entry. `settled` resolves once the child's executor has fully drained, * its reservation reconciled, and its result blob persisted; `next()` awaits these to drive @@ -148,11 +202,15 @@ interface LiveChild { runtime: NodeSnapshot['runtime'] readonly budget: Budget readonly label: string + readonly assignmentId?: string + readonly identity?: NodeExecutionIdentity /** The semantic spawn key, when this child was spawned with one — the settle path folds the * terminal state back into the scope's key registry under it. */ readonly key?: string spent: Spend outRef?: string + /** Exact terminal timestamp committed to the journal. */ + settledAt?: number /** Resolves with the terminal settlement WITHOUT a `seq` — `next()` stamps the seq. */ readonly settled: Promise /** Synchronous mirror of `settled`'s value once it has resolved (else `undefined`). */ @@ -161,6 +219,9 @@ interface LiveChild { * on `settled` resolves without further executor progress (only persistence/teardown), so a * non-blocking drain may await it without waiting on live work. */ executorDone: boolean + /** True only after executor teardown returned `{ destroyed: true }`. A failed/unknown cleanup + * retains the shared capacity slot so replacement work cannot exceed the physical live count. */ + cleanupConfirmed: boolean /** True once `next()` has yielded this child's settlement. */ delivered: boolean /** The executor's out-of-band inbox, captured at spawn — backs `scope.send`. */ @@ -169,6 +230,10 @@ interface LiveChild { readonly readProgress?: () => ExecutorProgress | undefined /** The executor's optional live tool trace, captured at spawn — backs `scope.traceSource`. */ readonly readTraceSource?: () => TraceSource | undefined + /** Kernel-owned declaration of the exact execution plan, durable before `execute` starts. */ + materialization?: ProfileMaterializationReceipt + /** One immutable record per concrete execution attempt. */ + executionBindings: ExecutionBindingReceipt[] /** Wall-clock of the spawn, and of the last metered usage event this child produced. Both are * stamped by the scope from the stream the conserved pool already meters, so EVERY executor — * including one that implements no progress read at all — has an observable liveness signal. */ @@ -176,7 +241,12 @@ interface LiveChild { lastActivityAt: number /** Present ONLY on a wait-state node. Its presence is what routes the settle path to the * `woken` journal event instead of `settled`, and what keeps a wait out of `inFlight`. */ - readonly wait?: { readonly spec: WaitSpec; readonly armedAt: number; readonly label: string } + readonly wait?: { + readonly spec: WaitSpec + readonly armedAt: number + readonly label: string + armCommitted: boolean + } } /** A child's terminal settlement before the cursor stamps the monotonic `seq`. A wait-state's @@ -206,7 +276,7 @@ type PreSeqSettled = /** * The recursion seam key. A `Scope` seeds a value of this on each child's * `ExecutorContext.seams` so a child whose executor is a DRIVER can mount a NESTED `Scope` - * over the SAME conserved pool at `depth+1`. A leaf executor never reads it. Single-sourced + * over the driver's reserved child allocation at `depth+1`. A leaf executor never reads it. Single-sourced * here so the scope and the driver-executor agree on the seam without a circular import. */ export const nestedScopeSeamKey = 'nested-scope' @@ -216,11 +286,13 @@ export const nestedScopeSeamKey = 'nested-scope' * driver child's own node id (so its children get `${nodeId}:s${ordinal}` ids and its * nested journal tree is namespaced under it); `root` is the journal tree key for the * nested tree (distinct from the parent's so cursor seqs never collide in the per-tree - * guard). `depth` is `parent.depth + 1`. The nested scope shares the parent's `pool` - * (conserved budget across depth), `journal`/`blobs` (one record), and `executors` (a - * nested child resolves to leaf-or-driver through the same open registry). + * guard). `depth` is `parent.depth + 1`. The nested scope spends from a child pool backed by + * the driver's already-reserved allocation; it shares the parent's `journal`/`blobs` and + * `executors` (a nested child resolves to leaf-or-driver through the same open registry). */ export interface NestedScopeSeam { + /** Durable id of the driver child that owns the nested tree. */ + readonly nodeId: NodeId /** This scope's recursion depth — a nested scope runs at `depth + 1`. */ readonly depth: number /** The runtime recursion-depth ceiling, paired with the conserved pool (R3). */ @@ -231,16 +303,35 @@ export interface NestedScopeSeam { mount(nestedRoot: NodeId, signal: AbortSignal): Scope } -function makeNestedScopeSeam(args: ScopeArgs, childNodeId: NodeId): NestedScopeSeam { +interface DeferredOwnerSlot { + ownerMaterialization?: NonNullable +} + +function makeNestedScopeSeam( + args: ScopeArgs, + liveWorkerCapacity: LiveWorkerCapacityState, + childNodeId: NodeId, + childBudget: Budget, + childDeadlineAtMs: number | undefined, + deferredOwner: DeferredOwnerSlot, +): NestedScopeSeam { + const now = args.now ?? Date.now return { + nodeId: childNodeId, depth: args.depth, ...(args.maxDepth !== undefined ? { maxDepth: args.maxDepth } : {}), journalRoot: args.root, mount(nestedRoot: NodeId, signal: AbortSignal): Scope { + const deadlineMs = + childDeadlineAtMs === undefined ? undefined : Math.max(0, childDeadlineAtMs - now()) + const nestedBudget = { + ...childBudget, + ...(deadlineMs !== undefined ? { deadlineMs } : {}), + } return createScope({ parentId: childNodeId, root: nestedRoot, - pool: args.pool, + pool: createBudgetPool(nestedBudget, now), journal: args.journal, blobs: args.blobs, executors: args.executors, @@ -249,9 +340,13 @@ function makeNestedScopeSeam(args: ScopeArgs, childNodeId: NodeId): NestedScopeS seams: args.seams, depth: args.depth + 1, ...(args.maxDepth !== undefined ? { maxDepth: args.maxDepth } : {}), + liveWorkerCapacity, signal, ...(args.now ? { now: args.now } : {}), ...(args.hooks ? { hooks: args.hooks } : {}), + ...(deferredOwner.ownerMaterialization === undefined + ? {} + : { ownerMaterialization: deferredOwner.ownerMaterialization }), }) }, } @@ -260,6 +355,10 @@ function makeNestedScopeSeam(args: ScopeArgs, childNodeId: NodeId): NestedScopeS /** Create the reactive `Scope` a driver's `Agent.act` runs inside: spawn children on an atomically reserved conserved budget, settle via the `next()` cursor, journal for replay. */ export function createScope(args: ScopeArgs): Scope { const children = new Map() + const liveWorkerCapacity: LiveWorkerCapacityState = args.liveWorkerCapacity ?? { + max: normalizeLiveWorkerLimit(args.maxLiveWorkers), + live: 0, + } // Two distinct monotonic counters in two namespaces: // - `spawnOrdinal` is the spawn order (0,1,2,…); it mints the deterministic node id // `${parent}:s${ordinal}` and stamps the `spawned` event's `seq`. Known at spawn. @@ -290,51 +389,101 @@ export function createScope(args: ScopeArgs): Scope { // committed result, `live` refuses a concurrent duplicate, `down`/`in-doubt` spawn fresh but // say so explicitly. type KeyState = - | { readonly state: 'live'; readonly id: NodeId } + | { readonly state: 'live'; readonly id: NodeId; readonly identity: NodeExecutionIdentity } | { readonly state: 'done' readonly id: NodeId + readonly identity: NodeExecutionIdentity readonly settled: Settled & { kind: 'done' } } - | { readonly state: 'down'; readonly id: NodeId; readonly reason: string } - | { readonly state: 'in-doubt'; readonly id: NodeId } + | { + readonly state: 'down' + readonly id: NodeId + readonly identity: NodeExecutionIdentity + readonly reason: string + } + | { readonly state: 'in-doubt'; readonly id: NodeId; readonly identity: NodeExecutionIdentity } const keyed = new Map() for (const [key, prior] of args.resumeFrom?.keys ?? []) { if (prior.state === 'completed' && prior.settled?.kind === 'done') { + if (prior.identity === undefined) continue keyed.set(key, { state: 'done', id: prior.id, + identity: prior.identity, settled: prior.settled as Settled & { kind: 'done' }, }) } else if (prior.state === 'down' && prior.settled?.kind === 'down') { - keyed.set(key, { state: 'down', id: prior.id, reason: prior.settled.reason }) - } else { - keyed.set(key, { state: 'in-doubt', id: prior.id }) + if (prior.identity === undefined) continue + keyed.set(key, { + state: 'down', + id: prior.id, + identity: prior.identity, + reason: prior.settled.reason, + }) + } else if (prior.identity !== undefined) { + keyed.set(key, { state: 'in-doubt', id: prior.id, identity: prior.identity }) } } /** Fold a keyed child's terminal settlement back into the registry, so a later spawn with the * same key resolves to it (done → committed result; down → explicit retry). */ const recordKeyedSettlement = (key: string, settled: Settled): void => { + const identity = settled.handle.identity + if (identity === undefined) { + throw new ValidationError(`scope: keyed settlement '${key}' lost its execution identity`) + } if (settled.kind === 'done') { - keyed.set(key, { state: 'done', id: settled.handle.id, settled }) + keyed.set(key, { state: 'done', id: settled.handle.id, identity, settled }) } else { - keyed.set(key, { state: 'down', id: settled.handle.id, reason: settled.reason }) + keyed.set(key, { state: 'down', id: settled.handle.id, identity, reason: settled.reason }) } } function spawn( - agent: Agent, - task: unknown, - opts: SpawnOpts, + agentOrFactory: Agent | (() => Agent), + rawTask: unknown, + rawOpts: SpawnOpts, ): | { ok: true; handle: Handle; prior?: SpawnPrior } | { ok: false; reason: SpawnRejection } { - // Resolve the semantic key FIRST — a committed key spends nothing (no reservation, no - // executor, no journal record: the prior settlement is already the durable record), and a - // still-live duplicate is refused before any resource is touched. + if (args.signal.aborted) return { ok: false, reason: 'scope-aborted' } + const task = detachedSnapshot(rawTask, 'scope.spawn task') + const opts = detachedSnapshot(rawOpts, 'scope.spawn options') + + // A key is an identity claim, not merely a cache label. On every reuse, prepare the requested + // agent far enough to derive the authorized profile/task identity, then compare it with the + // journal before returning an old result or retrying old work. No executor is resolved, + // constructed, reserved, or run on the completed path. let prior: SpawnPrior | undefined + let prepared: + | { + readonly agent: Agent + readonly spec: AgentSpec + readonly identity: NodeExecutionIdentity | undefined + } + | undefined + const prepare = () => { + const agent = typeof agentOrFactory === 'function' ? agentOrFactory() : agentOrFactory + const rawSpec = (agent as unknown as { executorSpec?: unknown }).executorSpec + if (!isAgentSpec(rawSpec)) { + throw new ValidationError( + `scope.spawn: agent "${agent.name}" exposes no \`executorSpec\` (AgentSpec) to resolve a Executor`, + ) + } + const spec = snapshotAgentSpec(rawSpec) + return { agent, spec, identity: deriveNodeExecutionIdentity(spec, task) } + } if (opts.key !== undefined) { const existing = keyed.get(opts.key) + if (existing !== undefined) { + prepared = prepare() + if (!isCompleteIdentity(prepared.identity)) { + return { ok: false, reason: 'invalid-identity' } + } + if (!sameNodeExecutionIdentity(existing.identity, prepared.identity)) { + return { ok: false, reason: 'key-conflict' } + } + } if (existing?.state === 'live') return { ok: false, reason: 'duplicate-key' } if (existing?.state === 'done') { return { @@ -354,61 +503,127 @@ export function createScope(args: ScopeArgs): Scope { return { ok: false, reason: 'depth-exceeded' } } - // Resolve the leaf executor through the OPEN registry FIRST (no reservation to unwind - // if the agent is misconfigured). An agent carries its executor mapping as the - // `executorSpec` (an `AgentSpec`); resolution precedence (BYO → router/inline → harness - // factory) lives in the registry, not in a call-site switch. - const spec = (agent as unknown as { executorSpec?: unknown }).executorSpec - if (!isAgentSpec(spec)) { - throw new ValidationError( - `scope.spawn: agent "${agent.name}" exposes no \`executorSpec\` (AgentSpec) to resolve a Executor`, - ) - } - const resolved = args.executors.resolve(spec) - if (!resolved.succeeded) throw new ValidationError(`scope.spawn: ${resolved.error}`) + // ONE admission counter is shared by the root scope and every recursive scope it mounts. + // Acquire before calling a lazy worker factory, resolving/constructing its executor, or + // reserving budget. A completed keyed assignment returned above never touches the counter. + const permit = acquireLiveWorker(liveWorkerCapacity) + if (!permit.ok) return { ok: false, reason: 'max-live-workers' } // Reserve the child's whole ceiling atomically; fail CLOSED when the pool can't cover - // it (never read-then-spawn overcommit, so Σk is conserved by construction). - const reservation = args.pool.reserve(opts.budget) - if (!reservation.ok) return { ok: false, reason: reservation.reason } + // it (never read-then-spawn overcommit, so Σk is conserved by construction). This happens + // before a fresh lazy agent factory is called: refused work constructs nothing. + let reservation: ReturnType + try { + reservation = args.pool.reserve(opts.budget) + } catch (error) { + permit.release() + throw error + } + if (!reservation.ok) { + permit.release() + return { ok: false, reason: reservation.reason } + } + + // Resolve the leaf executor through the open registry after both worker and budget admission. + // If preparation fails, refund the reservation here because runChild never receives it. + let spec: AgentSpec + let resolved: { succeeded: true; value: (spec: AgentSpec, ctx: ExecutorContext) => Executor } + let identity: NodeExecutionIdentity | undefined + try { + prepared ??= prepare() + spec = prepared.spec + identity = prepared.identity + if (opts.key !== undefined && !isCompleteIdentity(identity)) { + args.pool.reconcile(reservation.ticket, zeroSpend()) + permit.release() + return { ok: false, reason: 'invalid-identity' } + } + const outcome = args.executors.resolve(spec) + if (!outcome.succeeded) throw new ValidationError(`scope.spawn: ${outcome.error}`) + resolved = outcome + } catch (error) { + args.pool.reconcile(reservation.ticket, zeroSpend()) + permit.release() + throw error + } // Everything between reserve and runChild's hand-off owns the reservation. A SYNCHRONOUS // throw here (most likely the executor factory `resolved.value(spec, ctx)`) would otherwise // leak the reservation — runChild, which reconciles the ticket, is never reached. Release it // with zero spend on throw, then rethrow, so `total ≡ free + reserved + committed` holds. // (runChild is the last statement and never sync-throws, so there is no double-reconcile.) + let cascadeAbort: (() => void) | undefined + let clearChildDeadline: (() => void) | undefined try { const ordinal = spawnOrdinal++ const id: NodeId = `${args.parentId}:s${ordinal}` + const attemptId = newExecutionAttemptId(id) + const startedAt = now() + const childDeadlineAtMs = boundedChildDeadlineAt( + args.pool.readout().deadlineMs, + opts.budget.deadlineMs, + startedAt, + ) // The child's abort chains off this scope's signal (a scope abort reaps every child) - // AND off its own handle.abort(). Aborting mid-acquire cascades through the executor's - // signal into its acquireSandbox find-by-name reap, so an acquiring node never leaks. - const childAbort = new AbortController() - const cascadeAbort = () => childAbort.abort() - if (args.signal.aborted) childAbort.abort() + // AND off its own handle.abort() or bounded deadline. Aborting mid-acquire cascades through + // the executor's signal into its acquireSandbox find-by-name reap, so an acquiring node + // never leaks. + const controller = new AbortController() + cascadeAbort = () => controller.abort() + if (args.signal.aborted) controller.abort() else args.signal.addEventListener('abort', cascadeAbort, { once: true }) + if (childDeadlineAtMs !== undefined) { + clearChildDeadline = armDeadlineTimer(Math.max(0, childDeadlineAtMs - now()), () => + controller.abort('child deadline exceeded'), + ) + } // Seed THIS scope's own keystone deps into the child's `ExecutorContext.seams`, so a - // child whose executor is a DRIVER can mount a nested `Scope` at `depth+1` over the - // SAME conserved pool + shared journal/blobs/registry (the recursion seam). A leaf - // executor ignores it; the parent's sandbox/router seams still pass through for leaves. - // The mounted nested scope re-seeds the SAME bag for ITS children, so the recursion - // composes — a driver child of a driver child mounts one level deeper still. + // child whose executor is a DRIVER can mount a nested `Scope` at `depth+1` over a child + // pool backed by THIS reservation. A leaf executor ignores it; the parent's sandbox/router + // seams still pass through for leaves. Each nested driver repeats the same partitioning. + const deferredOwner: DeferredOwnerSlot = {} const ctx: ExecutorContext = { - signal: childAbort.signal, - seams: { ...args.seams, [nestedScopeSeamKey]: makeNestedScopeSeam(args, id) }, + signal: controller.signal, + node: { + rootId: args.root, + parentId: args.parentId, + nodeId: id, + attemptId, + ...(identity ? { identity } : {}), + }, + seams: { + ...args.seams, + [nestedScopeSeamKey]: makeNestedScopeSeam( + args, + liveWorkerCapacity, + id, + opts.budget, + childDeadlineAtMs, + deferredOwner, + ), + }, } const executor = resolved.value(spec, ctx) as Executor const handle: Handle = { id, label: opts.label, + ...(opts.assignmentId === undefined ? {} : { assignmentId: opts.assignmentId }), + ...(identity ? { identity } : {}), get status(): NodeStatus { return children.get(id)?.status ?? 'cancelled' }, + get materialization(): ProfileMaterializationReceipt | undefined { + return children.get(id)?.materialization + }, + get executionBindings(): ReadonlyArray | undefined { + const bindings = children.get(id)?.executionBindings + return bindings && bindings.length > 0 ? Object.freeze([...bindings]) : undefined + }, abort(reason?: string): void { - childAbort.abort(reason) + controller.abort(reason) }, } @@ -416,33 +631,113 @@ export function createScope(args: ScopeArgs): Scope { id, status: 'acquiring', runtime: executor.runtime, + ...(identity ? { identity } : {}), budget: opts.budget, label: opts.label, + ...(opts.assignmentId === undefined ? {} : { assignmentId: opts.assignmentId }), ...(opts.key !== undefined ? { key: opts.key } : {}), spent: zeroSpend(), settled: undefined as unknown as Promise, delivered: false, executorDone: false, - startedAt: now(), - lastActivityAt: now(), + cleanupConfirmed: false, + executionBindings: [], + startedAt, + lastActivityAt: startedAt, ...(executor.deliver ? { deliver: executor.deliver.bind(executor) } : {}), ...(executor.progress ? { readProgress: executor.progress.bind(executor) } : {}), ...(executor.traceSource ? { readTraceSource: executor.traceSource.bind(executor) } : {}), } children.set(id, live) - if (opts.key !== undefined) keyed.set(opts.key, { state: 'live', id }) + if (opts.key !== undefined) { + keyed.set(opts.key, { + state: 'live', + id, + identity: identity as NodeExecutionIdentity, + }) + } - void args.journal.appendEvent(args.root, { + const spawnCommitted = args.journal.appendEvent(args.root, { kind: 'spawned', id, parent: args.parentId, label: opts.label, ...(opts.key !== undefined ? { key: opts.key } : {}), + ...(opts.assignmentId === undefined ? {} : { assignmentId: opts.assignmentId }), budget: opts.budget, runtime: executor.runtime, + ...(identity ? { identity } : {}), seq: ordinal, at: new Date(now()).toISOString(), }) + const materializationCommitted = spawnCommitted.then(async () => { + const profileDigest = identity?.profileDigest ?? authoredProfileDigest(spec.profile) + let receipt: ProfileMaterializationReceipt + let binding: ExecutionBindingReceipt + const declaration = runtimeOwnedExecutorMaterialization(executor) + const deferredRuntime = runtimeOwnedDeferredExecutorRuntime(executor) + if (deferredRuntime !== undefined) { + deferredOwner.ownerMaterialization = { + runtime: deferredRuntime, + authoredProfile: spec.profile, + attemptId, + journalRoot: args.root, + nodeId: id, + requiredKnown: true, + onReceipt(materialization, executionBinding) { + live.runtime = materialization.runtime + live.materialization = materialization + live.executionBindings.push(executionBinding) + }, + } + return + } + if (declaration === undefined) { + receipt = unknownMaterializationReceipt({ + ...(profileDigest === undefined ? {} : { authoredProfileDigest: profileDigest }), + runtime: executor.runtime, + reason: 'executor-did-not-report', + }) + binding = unknownExecutionBindingReceipt(receipt, attemptId, 'executor-did-not-report') + } else { + try { + if (profileDigest === undefined) { + throw new ValidationError( + 'scope.spawn: a known materialization requires a canonical authored profile', + ) + } + receipt = knownMaterializationReceipt({ + authoredProfileDigest: profileDigest, + runtime: executor.runtime, + declaration, + }) + const reportedBinding = runtimeOwnedExecutorExecutionBinding(executor) + if (reportedBinding === undefined || reportedBinding.attemptId !== attemptId) { + throw new ValidationError( + 'scope.spawn: trusted executor did not bind the kernel-minted attempt id', + ) + } + binding = knownExecutionBindingReceipt(receipt, reportedBinding) + } catch (error) { + receipt = unknownMaterializationReceipt({ + ...(profileDigest === undefined ? {} : { authoredProfileDigest: profileDigest }), + runtime: executor.runtime, + reason: 'invalid-executor-report', + }) + binding = unknownExecutionBindingReceipt(receipt, attemptId, 'invalid-executor-report') + await appendNodeMaterialization(args, id, ordinal, receipt, binding, now) + live.materialization = receipt + live.executionBindings.push(binding) + throw new ValidationError( + `scope.spawn: executor ${JSON.stringify(executor.runtime)} returned invalid materialization evidence`, + { cause: error }, + ) + } + } + await appendNodeMaterialization(args, id, ordinal, receipt, binding, now) + live.materialization = receipt + live.executionBindings.push(binding) + }) notifyRuntimeHookEvent( args.hooks, @@ -457,7 +752,9 @@ export function createScope(args: ScopeArgs): Scope { payload: { childId: id, label: opts.label, + ...(opts.assignmentId === undefined ? {} : { assignmentId: opts.assignmentId }), runtime: executor.runtime, + ...(identity ? { identity } : {}), budget: opts.budget, depth: args.depth, }, @@ -471,25 +768,32 @@ export function createScope(args: ScopeArgs): Scope { const settled = runChild( live, executor, - childAbort, + controller, task, opts, args.pool, reservation.ticket, args.blobs, now, + materializationCommitted, + childDeadlineAtMs, ) .then((s) => { live.resolved = s return s }) .finally(() => { - args.signal.removeEventListener('abort', cascadeAbort) + if (live.cleanupConfirmed) permit.release() + clearChildDeadline?.() + if (cascadeAbort) args.signal.removeEventListener('abort', cascadeAbort) }) ;(live as { settled: Promise }).settled = settled return { ok: true, handle, ...(prior ? { prior } : {}) } } catch (err) { + permit.release() + clearChildDeadline?.() + if (cascadeAbort) args.signal.removeEventListener('abort', cascadeAbort) args.pool.reconcile(reservation.ticket, zeroSpend()) throw err } @@ -543,6 +847,7 @@ export function createScope(args: ScopeArgs): Scope { } function send(nodeId: NodeId, msg: unknown): boolean { + if (args.signal.aborted) return false const child = children.get(nodeId) // Deliver only to a child that is still LIVE (not yet yielded by the cursor) and whose executor // accepts an inbox. A settled/unknown child, or a leaf with no `deliver`, cannot be steered. @@ -568,6 +873,7 @@ export function createScope(args: ScopeArgs): Scope { spec: WaitSpec, opts: WaitOpts, ): { ok: true; handle: Handle } | { ok: false; reason: WaitRejection } { + if (args.signal.aborted) return { ok: false, reason: 'deadline-exceeded' } if (validateWaitSpec(spec) !== null) return { ok: false, reason: 'invalid-spec' } if (spec.kind === 'poll' && args.probes?.resolve(spec.probe) === undefined) { return { ok: false, reason: 'unknown-probe' } @@ -617,59 +923,71 @@ export function createScope(args: ScopeArgs): Scope { settled: undefined as unknown as Promise, delivered: false, executorDone: false, + cleanupConfirmed: true, + executionBindings: [], startedAt: armedAt, lastActivityAt: now(), - wait: { spec: effectiveSpec, armedAt, label: opts.label }, + wait: { + spec: effectiveSpec, + armedAt, + label: opts.label, + armCommitted: adopted !== undefined, + }, } children.set(id, live) // Only a FRESH arm journals `waiting`; an adopted one already has its record (re-writing it - // would duplicate the wait ordinal in the journal's per-tree guard). - if (!adopted) { - void args.journal.appendEvent(args.root, { - kind: 'waiting', - id, - parent: args.parentId, - label: opts.label, - spec: effectiveSpec, - armedAt, - seq: ordinal, - at: new Date(now()).toISOString(), - }) - } + // would duplicate the wait ordinal in the journal's per-tree guard). A fresh wait may not + // start racing its timer/probe until this identity record is durable: otherwise a zero-delay + // wait can journal `woken` before `waiting`, or disappear entirely if this append fails. + const armCommitted = adopted + ? Promise.resolve() + : args.journal.appendEvent(args.root, { + kind: 'waiting', + id, + parent: args.parentId, + label: opts.label, + spec: effectiveSpec, + armedAt, + seq: ordinal, + at: new Date(now()).toISOString(), + }) - notifyRuntimeHookEvent( - args.hooks, - { - id: `${id}:waiting`, - runId: args.root, - target: 'agent.spawn', - phase: 'after', - timestamp: now(), - stepIndex: ordinal, - parentId: args.parentId, - payload: { - childId: id, + const settled = armCommitted + .then(() => { + if (live.wait) live.wait.armCommitted = true + notifyRuntimeHookEvent( + args.hooks, + { + id: `${id}:waiting`, + runId: args.root, + target: 'agent.spawn', + phase: 'after', + timestamp: now(), + stepIndex: ordinal, + parentId: args.parentId, + payload: { + childId: id, + label: opts.label, + runtime: 'wait', + wait: effectiveSpec, + armedAt, + resumed: adopted !== undefined, + }, + }, + { signal: args.signal }, + ) + return runWait({ + spec: effectiveSpec, label: opts.label, - runtime: 'wait', - wait: effectiveSpec, armedAt, resumed: adopted !== undefined, - }, - }, - { signal: args.signal }, - ) - - const settled = runWait({ - spec: effectiveSpec, - label: opts.label, - armedAt, - resumed: adopted !== undefined, - signal: waitAbort.signal, - ...(args.probes ? { probes: args.probes } : {}), - now, - ...(args.waitSleep ? { sleep: args.waitSleep } : {}), - }) + signal: waitAbort.signal, + ...(args.probes ? { probes: args.probes } : {}), + now, + ...(args.waitSleep ? { sleep: args.waitSleep } : {}), + }) + }) .then(async (resolution): Promise => { live.executorDone = true live.lastActivityAt = now() @@ -738,10 +1056,21 @@ export function createScope(args: ScopeArgs): Scope { } async function meter(spend: Spend, detail?: Record): Promise { + if (args.signal.aborted) { + throw new ValidationError('scope.meter: cannot record new driver work after scope abort') + } const seq = meterSeq++ // Debit the driver's own inference against the shared conserved pool (free → committed), so // equal-k counts it live and `budget.tokensLeft` reflects it for the in-loop guard. - args.pool.observe(spend) + // An invalid observation (currently: unknown dollar cost under a dollar ceiling) still + // describes compute that already happened. Preserve it in the durable record before + // returning the refusal; otherwise the terminal result would falsely report that cost as $0. + let observeError: unknown + try { + args.pool.observe(spend) + } catch (error) { + observeError = error + } // Journal it as a `metered` event — the durable TWIN of the pool debit (as `settled` is the // twin of `reconcile`), so every journal-based cost reader sums driver inference automatically. // Awaited like the settled append (cost-critical), so it has landed before the supervisor's @@ -768,6 +1097,7 @@ export function createScope(args: ScopeArgs): Scope { }, { signal: args.signal }, ) + if (observeError !== undefined) throw observeError } // The replayed committed work, frozen once at construction — a resume-aware `act` reads it @@ -782,7 +1112,7 @@ export function createScope(args: ScopeArgs): Scope { } : undefined - return { + const scope: Scope = { spawn, next, nextResolved, @@ -799,7 +1129,277 @@ export function createScope(args: ScopeArgs): Scope { get budget() { return args.pool.readout() }, + get workerCapacity() { + return { + live: liveWorkerCapacity.live, + freeSlots: freeSlots(liveWorkerCapacity.live, liveWorkerCapacity.max), + } + }, } + if (args.ownerMaterialization !== undefined) { + const authoredProfile = + args.ownerMaterialization.authoredProfile === undefined + ? undefined + : detachedSnapshot( + args.ownerMaterialization.authoredProfile, + 'scope owner authored profile', + ) + ownerMaterializationStates.set(scope as Scope, { + journal: args.journal, + root: args.ownerMaterialization.journalRoot ?? args.root, + nodeId: args.ownerMaterialization.nodeId ?? args.parentId, + runtime: args.ownerMaterialization.runtime, + attemptId: args.ownerMaterialization.attemptId, + ...(authoredProfile === undefined ? {} : { authoredProfile }), + ...(authoredProfile === undefined + ? {} + : { authoredProfileDigest: authoredProfileDigest(authoredProfile) }), + ...(args.ownerMaterialization.prior === undefined + ? {} + : { prior: args.ownerMaterialization.prior }), + requiredKnown: args.ownerMaterialization.requiredKnown === true, + ...(args.ownerMaterialization.onReceipt === undefined + ? {} + : { onReceipt: args.ownerMaterialization.onReceipt }), + now, + receipt: args.ownerMaterialization.prior, + bindingPublished: false, + publishedThisProcess: false, + }) + } + return scope +} + +interface OwnerMaterializationState { + readonly journal: SpawnJournal + readonly root: NodeId + readonly nodeId: NodeId + readonly runtime: NodeSnapshot['runtime'] + readonly attemptId: string + readonly authoredProfile?: unknown + readonly authoredProfileDigest?: Sha256Digest + readonly prior?: ProfileMaterializationReceipt + readonly requiredKnown: boolean + readonly onReceipt?: ( + materialization: ProfileMaterializationReceipt, + binding: ExecutionBindingReceipt, + ) => void + readonly now: () => number + receipt?: ProfileMaterializationReceipt + bindingPublished: boolean + publishedThisProcess: boolean +} + +const ownerMaterializationStates = new WeakMap, OwnerMaterializationState>() + +/** + * @internal Publish exact root-manager materialization from a runtime-owned adapter after dynamic + * attachments exist and before its executor starts. This is deliberately a module function backed + * by a private WeakMap, not a Scope method an Agent can call. + */ +export async function recordScopeOwnerMaterialization( + scope: Scope, + runtime: NodeSnapshot['runtime'], + declaration: import('./types').ExecutorMaterialization, + bindingInput: ExecutorExecutionBinding, +): Promise { + const state = ownerMaterializationState(scope) + if (runtime !== state.runtime) { + await rejectOwnerMaterialization(state) + throw new ValidationError( + `scope owner materialization runtime ${JSON.stringify(runtime)} does not match ${JSON.stringify(state.runtime)}`, + ) + } + if (state.authoredProfileDigest === undefined) { + await rejectOwnerMaterialization(state) + throw new ValidationError('scope owner materialization requires an exact authored profile') + } + let receipt: ProfileMaterializationReceipt + let binding: ExecutionBindingReceipt + try { + if (bindingInput.attemptId !== state.attemptId) { + throw new ValidationError( + 'scope owner execution binding does not use the kernel-minted attempt id', + ) + } + if (canonicalCandidateDigest(declaration.effectiveProfile) !== state.authoredProfileDigest) { + throw new ValidationError( + 'scope owner stable effective profile conflicts with its admitted authored profile', + ) + } + receipt = knownMaterializationReceipt({ + authoredProfileDigest: state.authoredProfileDigest, + runtime, + declaration, + }) + binding = knownExecutionBindingReceipt(receipt, bindingInput) + } catch (error) { + await rejectOwnerMaterialization(state) + throw new ValidationError('scope owner returned invalid materialization evidence', { + cause: error, + }) + } + if (state.prior !== undefined) { + if (canonicalCandidateDigest(state.prior) !== canonicalCandidateDigest(receipt)) { + await rejectOwnerMaterialization(state) + throw new ValidationError( + 'scope owner materialization changed across resume; backend, model, execution identity, and plan must match', + ) + } + state.receipt = state.prior + await appendOwnerBinding(state, binding) + state.onReceipt?.(state.prior, binding) + state.publishedThisProcess = true + return + } + if (state.receipt !== undefined) { + throw new ValidationError('scope owner materialization was already recorded') + } + await appendOwnerMaterialization(state, receipt, binding) + state.onReceipt?.(receipt, binding) + state.publishedThisProcess = true +} + +/** @internal Kernel identity for constructing the exact deferred owner executor. */ +export function scopeOwnerExecutorNodeContext(scope: Scope): ExecutorNodeContext { + const state = ownerMaterializationState(scope) + return Object.freeze({ + rootId: state.root, + parentId: state.nodeId, + nodeId: state.nodeId, + attemptId: state.attemptId, + }) +} + +/** @internal Ensure a deferred root that never published evidence remains visibly unknown. */ +export async function finalizeScopeOwnerMaterialization(scope: Scope): Promise { + const state = ownerMaterializationStates.get(scope) + if (state === undefined || state.publishedThisProcess) return + if (state.prior !== undefined) { + await appendUnknownOwnerBinding(state, state.prior, 'root-agent-did-not-report') + throw new ValidationError( + 'resumed scope owner did not re-attest its prior materialization before execution', + ) + } + if (state.receipt !== undefined) return + const receipt = unknownMaterializationReceipt({ + ...(state.authoredProfileDigest === undefined + ? {} + : { authoredProfileDigest: state.authoredProfileDigest }), + runtime: state.runtime, + reason: 'root-agent-did-not-report', + }) + const binding = unknownExecutionBindingReceipt( + receipt, + state.attemptId, + 'root-agent-did-not-report', + ) + await appendOwnerMaterialization(state, receipt, binding) + state.onReceipt?.(receipt, binding) + if (state.requiredKnown) { + throw new ValidationError( + 'runtime-owned scope owner did not publish materialization before completing', + ) + } +} + +function ownerMaterializationState(scope: Scope): OwnerMaterializationState { + const state = ownerMaterializationStates.get(scope) + if (state === undefined) { + throw new ValidationError('scope has no deferred runtime-owned root materialization channel') + } + return state +} + +async function rejectOwnerMaterialization(state: OwnerMaterializationState): Promise { + if (state.bindingPublished) return + if (state.prior !== undefined) { + await appendUnknownOwnerBinding(state, state.prior, 'invalid-executor-report') + return + } + if (state.receipt !== undefined) { + await appendUnknownOwnerBinding(state, state.receipt, 'invalid-executor-report') + return + } + const receipt = unknownMaterializationReceipt({ + ...(state.authoredProfileDigest === undefined + ? {} + : { authoredProfileDigest: state.authoredProfileDigest }), + runtime: state.runtime, + reason: 'invalid-executor-report', + }) + const binding = unknownExecutionBindingReceipt( + receipt, + state.attemptId, + 'invalid-executor-report', + ) + await appendOwnerMaterialization(state, receipt, binding) + state.onReceipt?.(receipt, binding) +} + +async function appendOwnerMaterialization( + state: OwnerMaterializationState, + receipt: ProfileMaterializationReceipt, + binding: ExecutionBindingReceipt, +): Promise { + await state.journal.appendEvent(state.root, { + kind: 'materialized', + id: state.nodeId, + receipt, + seq: 0, + at: new Date(state.now()).toISOString(), + }) + state.receipt = receipt + await appendOwnerBinding(state, binding) +} + +async function appendOwnerBinding( + state: OwnerMaterializationState, + binding: ExecutionBindingReceipt, +): Promise { + await state.journal.appendEvent(state.root, { + kind: 'execution-bound', + id: state.nodeId, + binding, + seq: 0, + at: new Date(state.now()).toISOString(), + }) + state.bindingPublished = true +} + +async function appendUnknownOwnerBinding( + state: OwnerMaterializationState, + receipt: ProfileMaterializationReceipt, + reason: import('./types').UnknownMaterializationReason, +): Promise { + const binding = unknownExecutionBindingReceipt(receipt, state.attemptId, reason) + await appendOwnerBinding(state, binding) + state.onReceipt?.(receipt, binding) +} + +async function appendNodeMaterialization( + args: Pick, + id: NodeId, + seq: number, + receipt: ProfileMaterializationReceipt, + binding: ExecutionBindingReceipt, + now: () => number, +): Promise { + const at = new Date(now()).toISOString() + await args.journal.appendEvent(args.root, { + kind: 'materialized', + id, + receipt, + seq, + at, + }) + await args.journal.appendEvent(args.root, { + kind: 'execution-bound', + id, + binding, + seq, + at, + }) } /** Await whichever pending child settles first, returning the child (its `resolved` is set @@ -823,6 +1423,9 @@ async function finalizeSettlement( // journal reader can separate zero-cost waiting from paid work without inspecting payloads // (`spentFromJournal` therefore sums waits as the zero they are, with no special case). if (child.wait) return finalizeWait(child, settlement, seq, args, now, handle) + const settledAt = now() + child.settledAt = settledAt + const at = new Date(settledAt).toISOString() if (settlement.kind === 'down') { child.status = 'failed' await args.journal.appendEvent(args.root, { @@ -832,7 +1435,7 @@ async function finalizeSettlement( spent: child.spent, infra: settlement.infra, seq, - at: new Date(now()).toISOString(), + at, }) // Re-home a crashed driver child's partial inference too (the pool already debited it via // `observe`) — so spentTotal/trajectory never undercount a sub-driver that died mid-run. @@ -842,7 +1445,7 @@ async function finalizeSettlement( id: child.id, spend: settlement.metered, seq, - at: new Date(now()).toISOString(), + at, }) } notifyRuntimeHookEvent( @@ -852,7 +1455,7 @@ async function finalizeSettlement( runId: args.root, target: 'agent.child', phase: 'after', - timestamp: now(), + timestamp: settledAt, stepIndex: seq, parentId: args.parentId, payload: { @@ -871,6 +1474,7 @@ async function finalizeSettlement( reason: settlement.reason, infra: settlement.infra, restartCount: settlement.restartCount, + settledAt, seq, } } @@ -886,7 +1490,7 @@ async function finalizeSettlement( ...(settlement.verdict ? { verdict: settlement.verdict } : {}), spent: settlement.spent, seq, - at: new Date(now()).toISOString(), + at, }) // Re-home a driver child's OWN-inference subtree total up to THIS (parent) tree as a `metered` // event for the child node — mirroring how `settled.spent` rolls child WORK up. So summing any @@ -897,7 +1501,7 @@ async function finalizeSettlement( id: child.id, spend: settlement.metered, seq, - at: new Date(now()).toISOString(), + at, }) } notifyRuntimeHookEvent( @@ -907,7 +1511,7 @@ async function finalizeSettlement( runId: args.root, target: 'agent.child', phase: 'after', - timestamp: now(), + timestamp: settledAt, stepIndex: seq, parentId: args.parentId, payload: { @@ -928,6 +1532,7 @@ async function finalizeSettlement( outRef: settlement.outRef, ...(settlement.verdict ? { verdict: settlement.verdict } : {}), spent: settlement.spent, + settledAt, seq, } } @@ -943,22 +1548,29 @@ async function finalizeWait( now: () => number, handle: Handle, ): Promise> { - const at = new Date(now()).toISOString() + const settledAt = now() + child.settledAt = settledAt + const at = new Date(settledAt).toISOString() if (settlement.kind === 'down') { child.status = 'cancelled' - await args.journal.appendEvent(args.root, { - kind: 'woken', - id: child.id, - by: 'cancelled', - seq, - at, - }) + // A failed fresh `waiting` append means this node never existed durably. Do not leave a + // terminal `woken` record with no arm record for replay to attach it to. + if (child.wait?.armCommitted) { + await args.journal.appendEvent(args.root, { + kind: 'woken', + id: child.id, + by: 'cancelled', + seq, + at, + }) + } return { kind: 'down', handle, reason: settlement.reason, infra: settlement.infra, restartCount: settlement.restartCount, + settledAt, seq, } } @@ -980,7 +1592,7 @@ async function finalizeWait( runId: args.root, target: 'agent.child', phase: 'after', - timestamp: now(), + timestamp: settledAt, stepIndex: seq, parentId: args.parentId, payload: { childId: child.id, status: 'done', wait: out }, @@ -993,6 +1605,7 @@ async function finalizeWait( out: settlement.out as Out, outRef: settlement.outRef, spent: settlement.spent, + settledAt, seq, } } @@ -1018,17 +1631,44 @@ async function runChild( ticket: ReservationTicket, blobs: ResultBlobStore, now: () => number, + executionReady: Promise, + deadlineAtMs: number | undefined, ): Promise { let reconciled = false - const reconcileOnce = (spend: Spend) => { - if (reconciled) return + let started = false + let terminalTelemetryCaptured = false + let teardownStarted = false + const teardownOnce = async (grace: number | 'brutalKill' | 'infinity'): Promise => { + if (teardownStarted) return + teardownStarted = true + await teardownExecutor(executor, grace, deadlineAtMs, now) + live.cleanupConfirmed = true + } + const reconcileOnce = (spend: Spend): unknown | undefined => { + if (reconciled) return undefined reconciled = true - // A budgetExempt executor reports zero spend by contract; the reconcile refunds its - // whole reservation, keeping it out of the conserved Σk by construction. - pool.reconcile(ticket, clampSpend(spend, opts.budget)) + // A refused pre-execution path (including an unmetered executor) reconciles zero and refunds + // its whole reservation. Every path that actually executes reports measured or unknown spend. + try { + pool.reconcile(ticket, spend) + return undefined + } catch (error) { + return error + } } try { + // Identity and kernel-owned materialization evidence must be durable before execution can + // begin. A failed append or invalid executor declaration produces a typed-down child and + // refunds its reservation; the executor observes zero calls. + await executionReady + if (childAbort.signal.aborted) throw abortError(childAbort.signal) + if (executor.budgetExempt) { + throw new ValidationError( + `scope.spawn: runtime ${JSON.stringify(executor.runtime)} does not report usage and cannot execute inside a budgeted supervisor`, + ) + } live.status = 'running' + started = true const ran = executor.execute(task, childAbort.signal) let artifact: ExecutorResult if (isAsyncIterable(ran)) { @@ -1036,18 +1676,32 @@ async function runChild( // authority), then read the terminal artifact after the stream drains. Each event also // republishes the running total + a fresh activity stamp onto the live child, so a // concurrent `scope.progress(id)` sees a worker mid-flight rather than a zeroed row. - const spend = await foldStream(ran, (running) => { - live.spent = running - live.lastActivityAt = now() - }) + const spend = await foldStream( + ran, + (running) => { + live.spent = running + live.lastActivityAt = now() + }, + childAbort.signal, + ) live.spent = spend artifact = executor.resultArtifact() as ExecutorResult - reconcileOnce(spend) + const accounting = executor.accounting?.() + const terminalSpend = preserveUnknownTelemetry(spend, artifact.spent) + live.spent = accounting?.reported ?? terminalSpend + terminalTelemetryCaptured = true + live.executorDone = true + const reconcileError = reconcileOnce(accounting?.reservation ?? terminalSpend) + if (reconcileError !== undefined) throw reconcileError } else { - const terminal = await ran - live.spent = terminal.spent + const terminal = await awaitAbortable(Promise.resolve(ran), childAbort.signal) + const accounting = executor.accounting?.() + live.spent = accounting?.reported ?? terminal.spent artifact = terminal - reconcileOnce(terminal.spent) + terminalTelemetryCaptured = true + live.executorDone = true + const reconcileError = reconcileOnce(accounting?.reservation ?? terminal.spent) + if (reconcileError !== undefined) throw reconcileError } // Executor work is complete; everything below is persistence/teardown. From here `settled` // resolves without further executor progress — the non-blocking drain keys on this. @@ -1058,7 +1712,7 @@ async function runChild( const ownMetered = executor.metered?.() if (childAbort.signal.aborted) { - await teardownSafe(executor, opts.shutdown ?? 'brutalKill') + await teardownOnce(opts.shutdown ?? 'brutalKill') return downRecord('aborted before settle', true, ownMetered) } @@ -1070,7 +1724,7 @@ async function runChild( // so a crash never leaves a journaled ref pointing at a missing blob. const outRef = contentAddress(artifact.out) await blobs.put(outRef, artifact.out) - await teardownSafe(executor, opts.shutdown ?? 'infinity') + await teardownOnce(opts.shutdown ?? DEFAULT_SUCCESSFUL_SHUTDOWN_MS) return { kind: 'done', out: artifact.out, @@ -1083,12 +1737,32 @@ async function runChild( // A thrown executor has also finished its own work — only the down-record persistence // remains, so the non-blocking drain may await this child too. live.executorDone = true - // Reconcile the (likely partial) spend so the reservation is refunded even on a throw. - reconcileOnce(live.spent) - await teardownSafe(executor, 'brutalKill') + // A recursive executor can still report the nested work committed before it threw. + // Reconcile that whole partial subtree while journaling its child-work component separately. + let teardownError: unknown + try { + await teardownOnce('brutalKill') + } catch (error) { + teardownError = error + } + const accounting = executor.accounting?.() + if (accounting) live.spent = accounting.reported const aborted = childAbort.signal.aborted || isAbortError(err) + if (started && !terminalTelemetryCaptured && accounting === undefined) { + // The provider never delivered a terminal usage receipt. This is true for an ordinary + // network/provider crash just as it is for an abort. Preserve observed partial counts as a + // lower bound, but never reinterpret the unreported remainder as zero under either root + // ceiling. A recursive executor's explicit accounting remains authoritative on its throw + // path; a persistence/teardown failure after a terminal artifact does too. + live.spent = { ...live.spent, tokensKnown: false, usdKnown: false } + } + const reconcileError = reconcileOnce(accounting?.reservation ?? live.spent) // A crashed driver child still re-homes the partial inference it durably metered. - return downRecord(errMessage(err), aborted || isInfraError(err), executor.metered?.()) + return downRecord( + errMessage(teardownError ?? reconcileError ?? err), + teardownError !== undefined || reconcileError !== undefined || aborted || isInfraError(err), + executor.metered?.(), + ) } } @@ -1125,6 +1799,35 @@ export function settledToIteration(settled: Settled): Iteration void } | { ok: false } { + if (capacity.max !== undefined && capacity.live >= capacity.max) return { ok: false } + capacity.live += 1 + let released = false + return { + ok: true, + release(): void { + if (released) return + released = true + capacity.live -= 1 + if (capacity.live < 0) { + throw new ValidationError('scope: live-worker capacity released more than once') + } + }, + } +} + function makeTreeView(root: NodeId, children: Map): TreeView { const nodes: NodeSnapshot[] = [...children.values()].map((c) => ({ id: c.id, @@ -1133,7 +1836,14 @@ function makeTreeView(root: NodeId, children: Map): TreeView status: c.status, runtime: c.runtime, budget: c.budget, + ...(c.assignmentId === undefined ? {} : { assignmentId: c.assignmentId }), + ...(c.identity ? { identity: c.identity } : {}), + ...(c.materialization ? { materialization: c.materialization } : {}), + ...(c.executionBindings.length > 0 + ? { executionBindings: Object.freeze([...c.executionBindings]) } + : {}), spent: c.spent, + ...(c.settledAt === undefined ? {} : { settledAt: c.settledAt }), ...(c.outRef ? { outRef: c.outRef } : {}), })) return { @@ -1149,12 +1859,78 @@ function frozenHandle(child: LiveChild): Handle { id: child.id, label: child.label, status: child.status, + ...(child.assignmentId === undefined ? {} : { assignmentId: child.assignmentId }), + ...(child.identity ? { identity: child.identity } : {}), + ...(child.materialization ? { materialization: child.materialization } : {}), + ...(child.executionBindings.length > 0 + ? { executionBindings: Object.freeze([...child.executionBindings]) } + : {}), abort(): void { // A settled child is terminal; abort is a no-op (its executor already tore down). }, } } +/** Derive the portable identity of the exact profile and task a node will execute. Caller-owned + * candidate/correlation fields are checked at this boundary before they enter the durable log. */ +export function deriveNodeExecutionIdentity( + spec: Pick, + task: unknown, +): NodeExecutionIdentity | undefined { + const digest = (value: unknown) => { + try { + return canonicalCandidateDigest(value) + } catch { + return undefined + } + } + const profileDigest = digest(spec.profile) + const taskDigest = digest(task) + const candidateDigest = spec.execution?.candidateDigest + if (candidateDigest !== undefined && !sha256DigestSchema.safeParse(candidateDigest).success) { + throw new ValidationError('scope.spawn: execution.candidateDigest must be a sha256 digest') + } + const correlation = freezeCorrelation(spec.execution?.correlation) + if (!profileDigest && !taskDigest && !candidateDigest && !correlation) return undefined + return Object.freeze({ + ...(profileDigest ? { profileDigest } : {}), + ...(taskDigest ? { taskDigest } : {}), + ...(candidateDigest ? { candidateDigest } : {}), + ...(correlation ? { correlation } : {}), + }) +} + +function isCompleteIdentity( + identity: NodeExecutionIdentity | undefined, +): identity is NodeExecutionIdentity & { + readonly profileDigest: string + readonly taskDigest: string +} { + return identity?.profileDigest !== undefined && identity.taskDigest !== undefined +} + +function sameNodeExecutionIdentity(a: NodeExecutionIdentity, b: NodeExecutionIdentity): boolean { + return canonicalCandidateDigest(a) === canonicalCandidateDigest(b) +} + +function freezeCorrelation( + value: Readonly> | undefined, +): Readonly> | undefined { + if (value === undefined) return undefined + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new ValidationError('scope.spawn: execution.correlation must be a string record') + } + const entries = Object.entries(value) + for (const [key, item] of entries) { + if (key.length === 0 || typeof item !== 'string' || item.length === 0) { + throw new ValidationError( + 'scope.spawn: execution.correlation keys and values must be non-empty strings', + ) + } + } + return Object.freeze(Object.fromEntries(entries)) +} + /** * Fold a streaming executor's normalized usage into the conserved `Spend`, publishing the * running total after EVERY event via `onProgress`. @@ -1168,59 +1944,96 @@ function frozenHandle(child: LiveChild): Handle { async function foldStream( stream: AsyncIterable, onProgress?: (running: Spend) => void, + signal?: AbortSignal, ): Promise { const tokens = { input: 0, output: 0 } let usd = 0 let iterations = 0 - for await (const ev of stream) { - if (ev.kind === 'tokens') { - tokens.input += ev.input - tokens.output += ev.output - } else if (ev.kind === 'cost') { - usd += ev.usd - } else { - iterations += 1 + const iterator = stream[Symbol.asyncIterator]() + try { + for (;;) { + const next = signal + ? await awaitAbortable(Promise.resolve(iterator.next()), signal) + : await iterator.next() + if (next.done) break + const ev = next.value + if (ev.kind === 'tokens') { + tokens.input += ev.input + tokens.output += ev.output + } else if (ev.kind === 'cost') { + usd += ev.usd + } else { + iterations += 1 + } + onProgress?.({ iterations, tokens: { ...tokens }, usd, ms: 0 }) } - onProgress?.({ iterations, tokens: { ...tokens }, usd, ms: 0 }) + } catch (error) { + // Ask a cooperative async generator to close, but never let a broken `return()` hide the + // deadline that already won. Its promise is observed so a late rejection is not unhandled. + void Promise.resolve(iterator.return?.()).catch(() => undefined) + throw error } return { iterations, tokens, usd, ms: 0 } } -/** Clamp a child's reported spend to its reservation so the pool's fail-loud over-spend - * guard never trips on a benign overshoot from an external usage report; the difference - * refunds to the pool as if the child stopped at its ceiling. */ -function clampSpend(spend: Spend, budget: Budget): Spend { - const totalTokens = spend.tokens.input + spend.tokens.output - const tokensOk = totalTokens <= budget.maxTokens - const itersOk = spend.iterations <= budget.maxIterations - const usdOk = budget.maxUsd === undefined || spend.usd <= budget.maxUsd - if (tokensOk && itersOk && usdOk) return spend - const ratio = !tokensOk && totalTokens > 0 ? budget.maxTokens / totalTokens : 1 +/** Usage events carry measured increments; the terminal artifact carries whether a provider omitted + * a whole accounting channel. Preserve those unknowns on the common streaming path. */ +function preserveUnknownTelemetry(streamed: Spend, terminal: Spend): Spend { return { - iterations: Math.min(spend.iterations, budget.maxIterations), - tokens: - ratio < 1 - ? { - input: Math.floor(spend.tokens.input * ratio), - output: Math.floor(spend.tokens.output * ratio), - } - : spend.tokens, - usd: budget.maxUsd === undefined ? spend.usd : Math.min(spend.usd, budget.maxUsd), - ...(spend.usdKnown === false ? { usdKnown: false } : {}), - ms: spend.ms, + ...streamed, + ...(terminal.tokensKnown === false ? { tokensKnown: false } : {}), + ...(terminal.usdKnown === false ? { usdKnown: false } : {}), + ms: terminal.ms, } } -async function teardownSafe( - executor: Executor, - grace: number | 'brutalKill' | 'infinity', -): Promise { - try { - await executor.teardown(grace) - } catch { - // Teardown failure is observable through the node staying live; swallow so it never - // masks the settlement itself. The supervisor's join barrier reaps on its own grace. +async function awaitAbortable(work: Promise, signal: AbortSignal): Promise { + if (signal.aborted) { + // `execute()` may have returned a promise that rejects from the same abort. Observe it before + // returning the already-winning cancellation so a fast abort cannot become an unhandled + // provider rejection. + void work.catch(() => undefined) + throw abortError(signal) } + return await new Promise((resolve, reject) => { + let settled = false + const onAbort = () => { + // A cooperative executor often resolves its terminal usage receipt from an abort listener. + // Give that already-triggered resolution one microtask to win; an ignoring executor still + // loses immediately afterward. + queueMicrotask(() => { + if (settled) return + settled = true + cleanup() + reject(abortError(signal)) + }) + } + const cleanup = () => signal.removeEventListener('abort', onAbort) + signal.addEventListener('abort', onAbort, { once: true }) + work.then( + (value) => { + if (settled) return + settled = true + cleanup() + resolve(value) + }, + (error) => { + if (settled) return + settled = true + cleanup() + reject(error) + }, + ) + }) +} + +function abortError(signal: AbortSignal): Error { + const reason = signal.reason + const error = new Error( + typeof reason === 'string' && reason.length > 0 ? reason : 'execution aborted', + ) + error.name = 'AbortError' + return error } function downRecord(reason: string, infra: boolean, metered?: Spend): PreSeqSettled { @@ -1247,6 +2060,27 @@ function isAgentSpec(value: unknown): value is AgentSpec { return 'profile' in v && 'harness' in v } +/** Snapshot every data field whose bytes affect identity, admission, or materialization while + * preserving the executable callbacks by reference. The executor implementation is trusted code; + * its profile and attribution inputs are not. */ +function snapshotAgentSpec(raw: AgentSpec): AgentSpec { + const { + profile: rawProfile, + harness, + execution: rawExecution, + ...runtimeExtensions + } = raw as AgentSpec & Readonly> + const profile = detachedSnapshot(rawProfile, 'scope.spawn profile') + const execution = + rawExecution === undefined ? undefined : detachedSnapshot(rawExecution, 'scope.spawn execution') + return Object.freeze({ + ...runtimeExtensions, + profile, + harness, + ...(execution === undefined ? {} : { execution }), + }) as AgentSpec +} + function isAbortError(err: unknown): boolean { return ( typeof err === 'object' && diff --git a/src/runtime/supervise/snapshot.ts b/src/runtime/supervise/snapshot.ts new file mode 100644 index 00000000..2735f601 --- /dev/null +++ b/src/runtime/supervise/snapshot.ts @@ -0,0 +1,18 @@ +import { ValidationError } from '../../errors' + +/** Deeply detach and freeze untrusted data at a runtime decision boundary. The clone prevents the + * caller from mutating it later; the freeze prevents downstream code from mutating the snapshot. */ +export function detachedSnapshot(value: T, context: string): T { + try { + return deepFreeze(structuredClone(value)) + } catch (error) { + throw new ValidationError(`${context}: input must be structured-cloneable`, { cause: error }) + } +} + +function deepFreeze(value: T, seen = new Set()): T { + if (value === null || typeof value !== 'object' || seen.has(value)) return value + seen.add(value) + for (const child of Object.values(value as Record)) deepFreeze(child, seen) + return Object.freeze(value) +} diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index 84ce4deb..fde6ad75 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -7,30 +7,86 @@ * `workerFromBackend` derives the worker seam (`makeWorkerAgent`) from a backend config + an optional * completion oracle — so "where the workers run" is one data choice, not a hand-rolled factory. */ -import type { AgentProfile } from '@tangle-network/agent-interface' +import { randomUUID } from 'node:crypto' +import { resolve } from 'node:path' +import { + type AgentProfile, + type AgentProfileSecurityPolicy, + agentProfileSchema, + canonicalCandidateDigest, + type Sha256Digest, + validateAgentProfileSecurity, +} from '@tangle-network/agent-interface' +import type { BackendType } from '@tangle-network/sandbox' +import { + assertProfileMaterialization, + controlProfileMaterialization, + fullProfileMaterialization, + type ProfileMaterializationContract, + profileMaterializationAxes, + promptControlProfileMaterialization, + promptModelProfileMaterialization, + worktreeCliProfileMaterialization, +} from '../../agent/profile-materialization' import { ValidationError } from '../../errors' import type { AnalystRegistry, + AuthorizeDownMessage, + AuthorizedDownMessage, + CoordinationEvent, + DownMessageAuthorizationInput, MakeWorkerAgent, + WorkerSpawnContext, WorkerWatchOptions, } from '../../mcp/tools/coordination' import type { RouterConfig } from '../router-client' import type { ToolLoopChat, ToolLoopCompactionOptions } from '../tool-loop' +import { assertValidBudget, spendFromUsageEvents } from './budget' import { type DeliverableSpec, gateOnDeliverable } from './completion-gate' +import { DEFAULT_SUCCESSFUL_SHUTDOWN_MS, teardownExecutor } from './deadline' +import { driverChild } from './driver-executor' +import type { BusRecord } from './event-bus' import type { SupervisorFinalizer } from './finalizer' -import { assertModelAllowed } from './model-policy' +import { + attestRuntimeOwnedScopeOwner, + runtimeOwnedExecutorExecutionBinding, + runtimeOwnedExecutorMaterialization, + runtimeOwnedScopeOwnerRuntime, +} from './materialization' +import { assertModelAllowed, assertProfileModelsAllowed } from './model-policy' import { createFileRunContext, createInMemoryRunContext } from './run-context' -import { createExecutor, type ExecutorConfig } from './runtime' +import { + bindReusableExecutorExecutionId, + captureReusableExecutorConfig, + createExecutor, + type ExecutorConfig, + snapshotExecutorConfig, +} from './runtime' +import { + deriveNodeExecutionIdentity, + recordScopeOwnerMaterialization, + scopeOwnerExecutorNodeContext, +} from './scope' +import { detachedSnapshot } from './snapshot' import type { StopRule } from './stop-rules' import { createSupervisor } from './supervisor' -import { type DriveHarness, type SupervisorProfile, supervisorAgent } from './supervisor-agent' +import { + type DriveHarness, + type ResolveSupervisorTools, + type SupervisorNodeContext, + type SupervisorProfile, + supervisorAgent, +} from './supervisor-agent' import type { Agent, + AgentExecutionRef, AgentSpec, Budget, ExecutorContext, + NodeExecutionIdentity, ResultBlobStore, SpawnJournal, + UsageEvent, } from './types' import type { WaitProbeRegistry } from './wait' @@ -41,42 +97,433 @@ export function workerFromBackend( backend: ExecutorConfig, deliverable?: DeliverableSpec, ): MakeWorkerAgent { - return (rawProfile) => { - const p = (rawProfile ?? {}) as { name?: unknown } - const name = typeof p.name === 'string' && p.name.length > 0 ? p.name : 'worker' - // harness:null — createExecutor(backend) carries the harness in its config (the sandbox case-arm - // reads config.harness when the spec leaves it null); the BYO executor below resolves the leaf. - const spec: AgentSpec = { profile: rawProfile as AgentProfile, harness: null } - const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } - const built = createExecutor(backend)(spec, ctx) - const executor = deliverable ? gateOnDeliverable(built, deliverable) : built - return { name, act: async () => '', executorSpec: { ...spec, executor } } as Agent< - unknown, - unknown - > & { executorSpec: AgentSpec } + const capturedBackend = captureReusableExecutorConfig(backend, 'workerFromBackend') + const unscopedNamespace = randomUUID() + let unscopedOrdinal = 0 + return (rawProfile, spawnContext) => { + const parsed = agentProfileSchema.safeParse(rawProfile) + if (!parsed.success) { + throw new ValidationError(`workerFromBackend: invalid AgentProfile: ${parsed.error.message}`) + } + const profile = parsed.data + assertBackendProfileMaterialization(profile, capturedBackend, 'workerFromBackend') + const name = profile.name ?? 'worker' + // A Scope assignment is stable across reconstruction. Direct callers that omit that context + // still get isolation, but only Scope-backed calls claim durable external-session recovery. + const assignmentId = + spawnContext?.assignmentId ?? `unscoped:${unscopedNamespace}:${unscopedOrdinal++}` + const boundBackend = bindReusableExecutorExecutionId( + capturedBackend, + externalExecutionId('supervised-worker', { assignmentId }), + ) + const baseFactory = createExecutor(boundBackend) + // Carry the configured factory into Scope. It is built only AFTER reservation with the real + // child signal/context, so a rejected or already-completed keyed spawn creates no executor. + const executorFactory = (spec: AgentSpec, ctx: ExecutorContext) => { + const built = baseFactory(spec, ctx) + return deliverable ? gateOnDeliverable(built, deliverable) : built + } + const spec: AgentSpec = { + profile, + harness: null, + executorFactory, + ...(spawnContext?.execution ? { execution: spawnContext.execution } : {}), + } + return { name, act: async () => '', executorSpec: spec } as Agent & { + executorSpec: AgentSpec + } + } +} + +function externalExecutionId(kind: string, identity: unknown): string { + const digest = canonicalCandidateDigest({ kind, identity }) + return `${kind}-${digest.slice('sha256:'.length)}` +} + +function backendProfileMaterialization(backend: ExecutorConfig): ProfileMaterializationContract { + switch (backend.backend) { + case 'bridge': + case 'sandbox': + case 'provider': + return fullProfileMaterialization + case 'cli-worktree': + return backend.bridge ? fullProfileMaterialization : worktreeCliProfileMaterialization + case 'router': + case 'router-tools': + case 'pi': + return promptModelProfileMaterialization + case 'cli': + return controlProfileMaterialization + } +} + +function assertProfileContract( + profile: AgentProfile, + contract: ProfileMaterializationContract, + context: string, +): void { + assertProfileMaterialization({ + contract, + changedAxes: profileMaterializationAxes(profile), + context, + }) +} + +function assertBackendProfileMaterialization( + profile: AgentProfile, + backend: ExecutorConfig, + context: string, +): void { + assertProfileContract(profile, backendProfileMaterialization(backend), context) +} + +const coordinationMcpAlias = 'agent-runtime-coordination' +const defaultAllowedMcpHosts: string[] = [] +Object.freeze(defaultAllowedMcpHosts) + +/** Manager-authored profiles are untrusted until product policy says otherwise. Remote MCP and + * ambient connection grants therefore fail closed by default, in addition to local MCP and hooks. */ +export const DEFAULT_AUTHORED_PROFILE_SECURITY_POLICY: AgentProfileSecurityPolicy = Object.freeze({ + allowLocalMcp: false, + allowHooks: false, + allowedMcpHosts: defaultAllowedMcpHosts, + allowConnections: false, +}) + +function isExternalSupervisor(profile: AgentProfile): boolean { + return profile.harness !== undefined && profile.harness !== 'cli-base' +} + +function automaticDriverBackendSupported(backend: ExecutorConfig): boolean { + // The built-in coordination server binds host loopback. A local bridge can reach it; a remote + // sandbox cannot until the caller supplies an explicit relay/tunnel through `driveHarness`. + return backend.backend === 'bridge' +} + +function backendProfileOverlays(backend: ExecutorConfig | undefined): AgentProfile[] { + if (!backend) return [] + if (backend.backend === 'bridge' && backend.agentProfile) return [backend.agentProfile] + if (backend.backend === 'cli-worktree' && backend.bridge?.agentProfile) { + return [backend.bridge.agentProfile] } + return [] +} + +/** Run a harness-brained manager through the same executor factory as its children. The manager's + * full profile is preserved, the live coordination server is added under one reserved alias, and + * every streamed turn is charged to the manager's scope before it may continue. */ +function driveHarnessFromBackend( + backend: ExecutorConfig, + executionId: string, + now: () => number = Date.now, +): DriveHarness { + const capturedBackend = captureReusableExecutorConfig(backend, 'driveHarnessFromBackend') + const boundBackend = bindReusableExecutorExecutionId(capturedBackend, executionId) + const baseFactory = createExecutor(boundBackend) + const drive: DriveHarness = async ({ + profile, + task, + scope, + coordinationMcpUrl, + coordinationTools, + }) => { + const initialBudget = scope.budget + const hasLiveCoordination = scope.view.inFlight > 0 || scope.view.waiting > 0 + if ( + !hasLiveCoordination && + (initialBudget.tokensLeft <= 0 || + initialBudget.iterationsLeft <= 0 || + (initialBudget.usdCapped && initialBudget.usdLeft <= 0) || + (initialBudget.deadlineMs > 0 && now() >= initialBudget.deadlineMs)) + ) { + throw new ValidationError('driveHarnessFromBackend: supervisor budget exhausted') + } + if (profile.mcp?.[coordinationMcpAlias] !== undefined) { + throw new ValidationError( + `driveHarnessFromBackend: profile MCP alias ${JSON.stringify(coordinationMcpAlias)} is reserved`, + ) + } + const effectiveProfile = agentProfileSchema.parse({ + ...profile, + mcp: { + ...profile.mcp, + [coordinationMcpAlias]: { transport: 'http', url: coordinationMcpUrl }, + }, + }) + const stableCoordinationTools = detachedSnapshot( + coordinationTools, + 'driveHarnessFromBackend coordination tools', + ) + const spec: AgentSpec = { + profile: effectiveProfile, + harness: + boundBackend.backend === 'sandbox' + ? ((effectiveProfile.harness ?? boundBackend.harness ?? null) as BackendType | null) + : null, + } + const executor = baseFactory(spec, { + signal: scope.signal, + node: scopeOwnerExecutorNodeContext(scope), + seams: {}, + }) + let completed = false + let started = false + let terminalAccountingCaptured = false + let pendingUsage: UsageEvent[] = [] + let teardownStarted = false + const deadlineAtMs = scope.budget.deadlineMs || undefined + const teardownOnce = async (grace: number | 'brutalKill' | 'infinity') => { + if (teardownStarted) return + teardownStarted = true + await teardownExecutor(executor, grace, deadlineAtMs, now) + } + const meterPending = async () => { + if (pendingUsage.length === 0) return + const batch = pendingUsage + pendingUsage = [] + await scope.meter(spendFromUsageEvents(batch), { + role: 'driver', + runtime: executor.runtime, + }) + const budget = scope.budget + if ( + budget.tokensLeft <= 0 || + (budget.usdCapped && budget.usdLeft <= 0) || + (budget.deadlineMs > 0 && now() >= budget.deadlineMs) + ) { + throw new ValidationError('driveHarnessFromBackend: supervisor budget exhausted') + } + } + + let failed = false + let failure: unknown + try { + // Construction transfers cleanup ownership immediately. Even a rejected receipt or an + // unmetered runtime reaches the single bounded teardown path below. + const declaration = runtimeOwnedExecutorMaterialization(executor) + const executionBinding = runtimeOwnedExecutorExecutionBinding(executor) + if (declaration === undefined || executionBinding === undefined) { + throw new ValidationError( + `driveHarnessFromBackend: built-in runtime ${JSON.stringify(executor.runtime)} has no trusted materialization declaration or execution binding`, + ) + } + await recordScopeOwnerMaterialization( + scope, + executor.runtime, + { + ...declaration, + // The endpoint is one attempt's transport binding, not AgentProfile identity. Keep the + // admitted profile stable and commit the logical coordination capability separately. + effectiveProfile: profile, + platformAttachments: { + [coordinationMcpAlias]: { + kind: 'coordination-mcp', + transport: 'http', + tools: stableCoordinationTools, + }, + }, + }, + { + ...executionBinding, + binding: { + stableBinding: executionBinding.binding, + platformAttachments: { + [coordinationMcpAlias]: { + transport: 'http', + url: coordinationMcpUrl, + }, + }, + }, + descriptor: { + ...executionBinding.descriptor, + coordination: true, + }, + }, + ) + if (executor.budgetExempt) { + throw new ValidationError( + `driveHarnessFromBackend: runtime ${JSON.stringify(executor.runtime)} does not report usage and cannot drive a budgeted supervisor`, + ) + } + + started = true + const run = executor.execute(task, scope.signal) + if (isAsyncIterable(run)) { + for await (const event of run) { + if (event.kind === 'iteration') { + await meterPending() + } else { + pendingUsage.push(event) + } + } + await meterPending() + const artifact = executor.resultArtifact() + terminalAccountingCaptured = true + // A stream carries increments, while its terminal artifact says whether either accounting + // channel was omitted. Preserve unknowns in the shared pool instead of treating them as 0. + if (artifact.spent.tokensKnown === false || artifact.spent.usdKnown === false) { + await scope.meter( + { + iterations: 0, + tokens: { input: 0, output: 0 }, + ...(artifact.spent.tokensKnown === false ? { tokensKnown: false } : {}), + usd: 0, + ...(artifact.spent.usdKnown === false ? { usdKnown: false } : {}), + ms: 0, + }, + { role: 'driver', runtime: executor.runtime, telemetry: 'unknown' }, + ) + } + } else { + const artifact = await run + terminalAccountingCaptured = true + await scope.meter( + { ...artifact.spent, iterations: 0 }, + { role: 'driver', runtime: executor.runtime }, + ) + } + completed = true + } catch (error) { + failed = true + failure = error + } finally { + try { + await meterPending() + } catch (error) { + if (!failed) { + failed = true + failure = error + } + } + if (failed && started && !terminalAccountingCaptured) { + try { + await scope.meter( + { + iterations: 0, + tokens: { input: 0, output: 0 }, + tokensKnown: false, + usd: 0, + usdKnown: false, + ms: 0, + }, + { role: 'driver', runtime: executor.runtime, telemetry: 'unknown-after-failure' }, + ) + } catch (error) { + // The budget pool intentionally throws after durably recording unknown capped usage and + // closing that capacity. Only replace the original failure if the marker did not land. + const budget = scope.budget + if (budget.tokensKnown !== false || (budget.usdCapped && budget.usdKnown !== false)) { + failure = error + } + } + } + try { + await teardownOnce(completed ? DEFAULT_SUCCESSFUL_SHUTDOWN_MS : 'brutalKill') + } catch (error) { + if (!failed) { + failed = true + failure = error + } + } + } + if (failed) throw failure + } + return attestRuntimeOwnedScopeOwner(drive, 'cli') +} + +function isAsyncIterable(value: unknown): value is AsyncIterable { + return ( + value !== null && + typeof value === 'object' && + Symbol.asyncIterator in value && + typeof (value as AsyncIterable)[Symbol.asyncIterator] === 'function' + ) } export interface SuperviseOptions { /** The conserved compute pool for the whole run. */ readonly budget: Budget + /** Trusted candidate and pursuit attribution for the root. The runtime derives profile/task + * digests itself from the exact detached values it executes. */ + readonly execution?: AgentExecutionRef /** WHERE workers run — derives the worker seam. Provide this OR an explicit `makeWorkerAgent`. */ readonly backend?: ExecutorConfig /** The completion oracle for backend-derived workers (settled ⟺ delivered). Strongly recommended: * without it the supervisor trusts a worker's self-report — exactly the "ran but didn't deliver" * failure mode of a static orchestrator. */ readonly deliverable?: DeliverableSpec - /** Override the worker seam directly (tests / advanced) instead of deriving it from `backend`. */ + /** Override the worker seam directly (tests / advanced) instead of deriving it from `backend`. + * This is caller-owned execution: profile security, spawn authorization, and recursive-driver + * selection below apply only to the backend-derived worker path. `authorizeMessage` still + * governs continuations sent through Runtime's coordination tools. */ readonly makeWorkerAgent?: MakeWorkerAgent - /** The supervisor's router substrate (`harness` null). The profile's model wins. */ + /** Run harness-brained supervisors here. Automatic execution supports a local `bridge`; a remote + * sandbox requires an explicit `driveHarness` with a reachable coordination relay or tunnel. + * Defaults to `backend`; separate it when managers and workers use different services. */ + readonly driverBackend?: ExecutorConfig + /** Security policy applied to every manager-authored child profile before budget reservation. + * The default blocks local and remote MCP, hooks, and connection grants. Pass an explicit + * allowlist to grant remote MCP hosts or other author-controlled capabilities. */ + readonly profileSecurity?: AgentProfileSecurityPolicy + /** Product authority over one complete manager-authored spawn. The callback sees the detached, + * immutable profile, task, budget, label, and key together, so approving a profile cannot + * authorize a different task. Return the exact allowed profile (which may be narrowed) plus + * trusted candidate/pursuit attribution, or throw to refuse the whole spawn before reservation. */ + readonly authorizeSpawn?: (input: { + readonly profile: AgentProfile + readonly parent: AgentProfile + /** Trusted identity of the manager authorizing this exact child. */ + readonly parentIdentity: NodeExecutionIdentity + /** Concrete manager node; never accepted from model-authored tool arguments. */ + readonly parentNodeId: string + /** Stable manager-scoped assignment, including deterministic unkeyed siblings. */ + readonly assignmentId: string + readonly task: unknown + readonly budget: Budget + readonly label: string + readonly key?: string + readonly depth: number + }) => AuthorizedSpawn + /** Product authority over every continuation sent to a live child. When spawn authorization is + * enabled, omitting this refuses steer/answer instructions instead of silently extending the + * authorized task. The exact worker identity and detached bytes are recorded before delivery. */ + readonly authorizeMessage?: ( + input: DownMessageAuthorizationInput & { + readonly parent: AgentProfile + readonly depth: number + }, + ) => AuthorizedDownMessage + /** Decide whether an authorized child becomes another supervisor. By default only + * `metadata.role === 'driver'` does; products may bind this to their own authority record. */ + readonly isDriverProfile?: (input: { + readonly profile: AgentProfile + readonly parent: AgentProfile + readonly depth: number + }) => boolean + /** The supervisor's router substrate (`profile.harness` omitted or `cli-base`). The profile's + * model wins. */ readonly router?: RouterConfig /** Inject the supervisor brain directly (tests / advanced). */ readonly brain?: ToolLoopChat - /** Run a sandboxed-harness supervisor (`harness` set). */ + /** Run an external-harness supervisor explicitly. Required for a remote sandbox; optional as a + * caller-owned override for a local bridge. */ readonly driveHarness?: DriveHarness + /** Required with a custom `driveHarness`: declares which complete AgentProfile axes that path + * really applies. Built-in bridge driving supplies its own full-profile contract. */ + readonly driveHarnessMaterialization?: ProfileMaterializationContract + /** Resolve product-owned tools from the exact trusted manager context. The same descriptors and + * handlers are bound to router and external-harness managers; resolution happens once per node. */ + readonly resolveSupervisorTools?: ResolveSupervisorTools + /** Awaited product transaction hook for every coordination record. `eventId` is stable across a + * lost acknowledgement and durable restart; the record is not pull-visible until this commits. */ + readonly onCoordinationEvent?: ( + context: SupervisorNodeContext, + eventId: Sha256Digest, + record: BusRecord, + ) => void | Promise /** WORK tools the supervisor may call DIRECTLY — so a recursive atom can ACT (do simple work * itself) OR SPAWN (delegate when it needs parallelism), not be a pure manager. Pair with - * `executeExtraTool`. Router arm only (`harness` null). */ + * `executeExtraTool`. Router arm only (`profile.harness` omitted or `cli-base`). */ readonly extraTools?: ReadonlyArray<{ readonly name: string readonly description?: string @@ -89,9 +536,9 @@ export interface SuperviseOptions { ) => Promise /** Per-child budget reserved on each spawn. Defaults to a quarter of the pool's tokens. */ readonly perWorker?: Budget - /** Hard cap on simultaneously-LIVE workers — `spawn_agent` fails closed once this many are in - * flight. The conserved pool bounds TOTAL work; this bounds SIMULTANEOUS work (live boxes/ - * sandboxes a real fleet runs at once). Omit/`<= 0` = no cap (the pool stays the only fence). */ + /** Hard cap on simultaneously executing spawned workers across the WHOLE recursive tree. The + * root is excluded; nested drivers and leaves share one allocation, so recursion cannot multiply + * the cap. Omit/`<= 0` = no cap (the conserved pool stays the only bound). */ readonly maxLiveWorkers?: number /** Analyst lenses available to the driver. Required for `analyzeOnSettle`. Unset → status quo * (the driver receives settled worker outputs, no analyst findings). */ @@ -119,17 +566,23 @@ export interface SuperviseOptions { /** * Make the run DURABLE: journal + result blobs + the coordination side-log are file-backed under * this directory (`createFileRunContext`), fsynced per write, and the supervisor reads the prior - * tree first. Re-running with the same `runDir` AND the same `runId` resumes, and the built-in - * driver is resume-AWARE out of the box: the children that already settled are replayed onto + * tree first. Re-running with the same `runDir` AND the same `runId` resumes only when the exact + * root profile/task identity and declared budget match. The original absolute deadline and prior + * measured spend are restored before new admission. The built-in driver is resume-aware: children + * that already settled, including their exact execution identities, are replayed onto * `Scope.resume` (and into the driver's settled ledger + its first context), keyed assignments * (`spawn_agent`'s `key`) resolve to their committed results instead of re-running, pending - * waits re-arm on their original deadlines, prior questions/findings replay from the - * coordination log, and the finalize spans both processes' work. Unset = in-memory, fresh - * every call. + * waits re-arm on their original deadlines, and the coordination log loads prior questions, + * findings, and instruction receipts. The router arm receives all three in its resume brief; the + * external arm seeds prior questions while findings and receipts remain in the durable log. + * Instruction receipts are evidence and are never delivered automatically to a replacement + * worker. The final result spans both processes' work. Unset = in-memory, fresh every call. * * The boundary that remains: work that was IN FLIGHT when the process died is not recovered — - * the built-in executors cannot re-attach to a dead process's executions, so those assignments - * resume as explicitly lost/in-doubt and re-run (reported, never silent). + * the built-in executors cannot re-attach to a dead process's executions. Each such assignment + * resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively, + * and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining + * capacity, so restart cannot mint a fresh budget or slide the original absolute deadline. * * `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single * resumable run per directory but collides across concurrent runs sharing one `runDir`. @@ -178,95 +631,610 @@ export interface SuperviseOptions { readonly finalizer?: SupervisorFinalizer } -/** A quarter of the token pool per worker → ~4 workers fit before `poolStarved` halts spawning. */ +/** The product-authorized result for one complete spawn request. Attribution is never accepted + * from the manager itself; it enters only through this trusted callback. */ +export interface AuthorizedSpawn { + readonly profile: AgentProfile + readonly execution?: AgentExecutionRef +} + +/** Capture the public one-call configuration before any asynchronous work starts. Decision data is + * detached and frozen; executable ports are copied as the exact references selected at intake. + * Service internals intentionally remain live, while replacing a callback/service on the caller's + * mutable options object can no longer change an in-flight run. */ +function captureSuperviseOptions(opts: SuperviseOptions): SuperviseOptions { + const { + backend, + driverBackend, + deliverable, + router, + compaction, + watchWorkers, + analysts, + makeWorkerAgent, + blobs, + journal, + probes, + authorizeSpawn, + authorizeMessage, + isDriverProfile, + brain, + driveHarness, + resolveSupervisorTools, + onCoordinationEvent, + executeExtraTool, + stopRule, + onProgressStop, + finalizer, + now, + ...decisionData + } = opts + const capturedData = detachedSnapshot(decisionData, 'supervise options') + const capturedBackend = backend === undefined ? undefined : snapshotExecutorConfig(backend) + const capturedDriverBackend = + driverBackend === undefined ? undefined : snapshotExecutorConfig(driverBackend) + const capturedDeliverable = + deliverable === undefined + ? undefined + : Object.freeze({ + ...detachedSnapshot( + { describe: deliverable.describe }, + 'supervise deliverable configuration', + ), + check: deliverable.check, + }) + const capturedRouter = + router === undefined + ? undefined + : (() => { + const { complete, ...routerData } = router + return Object.freeze({ + ...detachedSnapshot(routerData, 'supervise router configuration'), + ...(complete === undefined ? {} : { complete }), + }) + })() + const capturedCompaction = + compaction === undefined + ? undefined + : (() => { + const { distill, estimateTokens, onCompact, ...compactionData } = compaction + return Object.freeze({ + ...detachedSnapshot(compactionData, 'supervise compaction configuration'), + ...(distill === undefined ? {} : { distill }), + ...(estimateTokens === undefined ? {} : { estimateTokens }), + ...(onCompact === undefined ? {} : { onCompact }), + }) + })() + const capturedWatchWorkers = + watchWorkers === undefined + ? undefined + : Object.freeze({ + ...detachedSnapshot( + { maxFindingsPerWorker: watchWorkers.maxFindingsPerWorker }, + 'supervise worker-watch configuration', + ), + ...(watchWorkers.detectors === undefined + ? {} + : { detectors: Object.freeze([...watchWorkers.detectors]) }), + }) + const capturedAnalysts = + analysts === undefined + ? undefined + : Object.freeze({ + kinds: detachedSnapshot(analysts.kinds, 'supervise analyst kinds'), + run: analysts.run, + }) + + return Object.freeze({ + ...capturedData, + ...(capturedBackend === undefined ? {} : { backend: capturedBackend }), + ...(capturedDriverBackend === undefined ? {} : { driverBackend: capturedDriverBackend }), + ...(capturedDeliverable === undefined ? {} : { deliverable: capturedDeliverable }), + ...(capturedRouter === undefined ? {} : { router: capturedRouter }), + ...(capturedCompaction === undefined ? {} : { compaction: capturedCompaction }), + ...(capturedWatchWorkers === undefined ? {} : { watchWorkers: capturedWatchWorkers }), + ...(capturedAnalysts === undefined ? {} : { analysts: capturedAnalysts }), + ...(makeWorkerAgent === undefined ? {} : { makeWorkerAgent }), + ...(blobs === undefined ? {} : { blobs }), + ...(journal === undefined ? {} : { journal }), + ...(probes === undefined ? {} : { probes }), + ...(authorizeSpawn === undefined ? {} : { authorizeSpawn }), + ...(authorizeMessage === undefined ? {} : { authorizeMessage }), + ...(isDriverProfile === undefined ? {} : { isDriverProfile }), + ...(brain === undefined ? {} : { brain }), + ...(driveHarness === undefined ? {} : { driveHarness }), + ...(resolveSupervisorTools === undefined ? {} : { resolveSupervisorTools }), + ...(onCoordinationEvent === undefined ? {} : { onCoordinationEvent }), + ...(executeExtraTool === undefined ? {} : { executeExtraTool }), + ...(stopRule === undefined ? {} : { stopRule }), + ...(onProgressStop === undefined ? {} : { onProgressStop }), + ...(finalizer === undefined ? {} : { finalizer }), + ...(now === undefined ? {} : { now }), + }) +} + +/** A quarter of token and optional dollar capacity per worker; nested managers partition again. */ function defaultPerWorker(budget: Budget): Budget { return { - maxIterations: budget.maxIterations, + maxIterations: Math.max(1, Math.floor(budget.maxIterations / 4)), maxTokens: Math.max(1, Math.floor(budget.maxTokens / 4)), + ...(budget.maxUsd !== undefined ? { maxUsd: budget.maxUsd / 4 } : {}), } } +function freezeDetached(value: T): T { + return detachedSnapshot(value, 'supervise') +} + +function freezeDetachedProfile(value: unknown): AgentProfile { + return freezeDetached(agentProfileSchema.parse(value)) +} + +function canonicalExecution( + profile: AgentProfile, + task: unknown, + rawExecution: AgentExecutionRef | undefined, + context: string, +): { readonly identity: NodeExecutionIdentity; readonly ref?: AgentExecutionRef } { + const execution = rawExecution === undefined ? undefined : freezeDetached(rawExecution) + if (execution !== undefined) { + if (typeof execution !== 'object' || execution === null || Array.isArray(execution)) { + throw new ValidationError(`${context}: execution must be an object`) + } + const unknown = Object.keys(execution).filter( + (key) => key !== 'candidateDigest' && key !== 'correlation', + ) + if (unknown.length > 0) { + throw new ValidationError(`${context}: unknown execution fields: ${unknown.join(', ')}`) + } + } + const identity = deriveNodeExecutionIdentity({ profile, execution }, task) + if (!identity?.profileDigest || !identity.taskDigest) { + throw new ValidationError( + `${context}: profile and task must be finite, acyclic canonical JSON for durable identity`, + ) + } + const ref: AgentExecutionRef | undefined = + identity.candidateDigest || identity.correlation + ? Object.freeze({ + ...(identity.candidateDigest ? { candidateDigest: identity.candidateDigest } : {}), + ...(identity.correlation ? { correlation: identity.correlation } : {}), + }) + : undefined + return { identity, ...(ref ? { ref } : {}) } +} + +function rootCoordinationOwner(identity: NodeExecutionIdentity): string { + return canonicalCandidateDigest({ kind: 'supervisor-root', identity }) +} + +function childCoordinationOwner( + parentOwnerId: string, + identity: NodeExecutionIdentity, + context: WorkerSpawnContext, + depth: number, +): string { + return canonicalCandidateDigest({ + kind: 'supervisor-child', + parentOwnerId, + identity, + assignment: { + id: context.assignmentId, + label: context.label, + key: context.key ?? null, + depth, + }, + }) +} + +function supervisionRunNamespace(runDir: string | undefined, runId: string): string { + return canonicalCandidateDigest( + runDir === undefined + ? { kind: 'supervise-ephemeral-run', runId, nonce: randomUUID() } + : { kind: 'supervise-durable-run', runId, runDir: resolve(runDir) }, + ) +} + +function workerAssignmentNamespace( + runNamespace: string, + parentOwnerId: string, + assignmentId: string, +): string { + return canonicalCandidateDigest({ + kind: 'supervise-worker-assignment', + runNamespace, + parentOwnerId, + assignmentId, + }) +} + +/** Hash only durable coordination meaning. Bus sequence/timestamp are delivery metadata and a + * resumed projection's marker describes the reader, not the original settlement. */ +function coordinationEventId( + context: SupervisorNodeContext, + event: CoordinationEvent, +): Sha256Digest { + const durableEvent = + event.type === 'settled' && event.worker.resumed === true + ? (() => { + const { resumed: _resumed, ...worker } = event.worker + return { type: 'settled' as const, worker } + })() + : event + return canonicalCandidateDigest({ + kind: 'supervise-coordination-event', + runNamespace: context.runNamespace, + ownerId: context.ownerId, + event: detachedSnapshot(durableEvent, 'supervise coordination event identity'), + }) +} + /** One-call supervisor: build + run a supervisor from its profile with sensible defaults; the raw `supervisorAgent` + `createSupervisor().run` seams stay available for power use. */ export function supervise(profile: SupervisorProfile, task: unknown, opts: SuperviseOptions) { + const options = captureSuperviseOptions(opts) + assertValidBudget(options.budget, 'supervise budget') // Fail loud before any compute: every configured model must be in the allowed subset (no-op // when allowedModels is unset). The backend seam carries its own model on most backends. - const backendModel = (opts.backend as { model?: unknown } | undefined)?.model - assertModelAllowed(opts.router?.model, opts.allowedModels) - assertModelAllowed(profile.model, opts.allowedModels) + const parsedProfile = agentProfileSchema.safeParse(profile) + if (!parsedProfile.success) { + throw new ValidationError(`supervise: invalid AgentProfile: ${parsedProfile.error.message}`) + } + const canonicalProfile = freezeDetachedProfile(parsedProfile.data) + const canonicalTask = freezeDetached(task) + if (options.makeWorkerAgent && options.authorizeSpawn) { + throw new ValidationError( + 'supervise: authorizeSpawn cannot be combined with caller-owned makeWorkerAgent; wrap and authorize the custom factory explicitly or use backend-derived workers', + ) + } + const authorizeDownFor = ( + parent: AgentProfile, + depth: number, + ): AuthorizeDownMessage | undefined => { + if (!options.authorizeSpawn && !options.authorizeMessage) return undefined + return (input) => { + if (!options.authorizeMessage) { + throw new ValidationError( + 'supervise: authorizeMessage is required before steer_agent or answer_question when authorizeSpawn is enabled', + ) + } + return freezeDetached( + options.authorizeMessage( + freezeDetached({ + ...input, + parent, + depth, + }), + ), + ) + } + } + const rootExecution = canonicalExecution( + canonicalProfile, + canonicalTask, + options.execution, + 'supervise root', + ) + const backendModel = (options.backend as { model?: unknown } | undefined)?.model + const driverBackendModel = (options.driverBackend as { model?: unknown } | undefined)?.model + const overlays = [ + ...backendProfileOverlays(options.backend), + ...backendProfileOverlays(options.driverBackend), + ] + if (overlays.length > 0) { + throw new ValidationError( + 'supervise: backend agentProfile overlays are not allowed because they run after spawn authorization; merge the overlay into the exact profile before calling supervise', + ) + } + assertModelAllowed(options.router?.model, options.allowedModels) + assertProfileModelsAllowed(canonicalProfile, options.allowedModels) assertModelAllowed( typeof backendModel === 'string' ? backendModel : undefined, - opts.allowedModels, + options.allowedModels, + ) + assertModelAllowed( + typeof driverBackendModel === 'string' ? driverBackendModel : undefined, + options.allowedModels, ) // `withDriver: true` is the wiring invariant either way (a `role: 'driver'` child must resolve // to the nested-scope executor); `runDir` only changes WHERE the journal and blobs live. const ctx = - opts.runDir !== undefined - ? createFileRunContext(opts.runDir, { withDriver: true }) + options.runDir !== undefined + ? createFileRunContext(options.runDir, { withDriver: true }) : createInMemoryRunContext({ withDriver: true }) - const blobs = opts.blobs ?? ctx.blobs - const perWorker = opts.perWorker ?? defaultPerWorker(opts.budget) + const blobs = options.blobs ?? ctx.blobs + const perWorker = options.perWorker ?? defaultPerWorker(options.budget) + assertValidBudget(perWorker, 'supervise perWorker') + const journal = options.journal ?? ctx.journal + const runId = options.runId ?? 'supervise' + const runNamespace = supervisionRunNamespace(options.runDir, runId) + const log = ctx.coordinationLog + const rootOwnerId = rootCoordinationOwner(rootExecution.identity) + const observeNodeEvent = options.onCoordinationEvent + ? async ( + context: SupervisorNodeContext, + event: CoordinationEvent, + record: BusRecord, + ) => { + await options.onCoordinationEvent?.(context, coordinationEventId(context, event), record) + } + : undefined + const managerBackend = options.driverBackend ?? options.backend + if (options.driveHarness && !options.driveHarnessMaterialization) { + throw new ValidationError( + 'supervise: custom driveHarness requires driveHarnessMaterialization so profile fields cannot be silently dropped', + ) + } + const driverMaterialization = options.driveHarness + ? options.driveHarnessMaterialization + : managerBackend && automaticDriverBackendSupported(managerBackend) + ? backendProfileMaterialization(managerBackend) + : undefined + if ( + isExternalSupervisor(canonicalProfile) && + !options.driveHarness && + (!managerBackend || !automaticDriverBackendSupported(managerBackend)) + ) { + throw new ValidationError( + `supervise: external supervisor profile.harness=${JSON.stringify(canonicalProfile.harness)} requires a local bridge driverBackend or an explicit driveHarness with reachable coordination transport`, + ) + } + const driveHarnessForOwner = (ownerId: string): DriveHarness | undefined => + options.driveHarness ?? + (managerBackend && automaticDriverBackendSupported(managerBackend) + ? driveHarnessFromBackend( + managerBackend, + externalExecutionId('supervised-manager', { runNamespace, ownerId }), + options.now ?? Date.now, + ) + : undefined) + const rootDriveHarness = driveHarnessForOwner(rootOwnerId) + const rootOwnerRuntime = + !isExternalSupervisor(canonicalProfile) || rootDriveHarness === undefined + ? undefined + : runtimeOwnedScopeOwnerRuntime(rootDriveHarness) + assertProfileContract( + canonicalProfile, + isExternalSupervisor(canonicalProfile) + ? (driverMaterialization as ProfileMaterializationContract) + : options.brain + ? promptControlProfileMaterialization + : promptModelProfileMaterialization, + 'supervise root', + ) - let makeWorkerAgent = opts.makeWorkerAgent + let makeWorkerAgent = options.makeWorkerAgent if (!makeWorkerAgent) { - if (!opts.backend) { + if (!options.backend) { throw new ValidationError( 'supervise: provide opts.backend (where workers run) or opts.makeWorkerAgent', ) } - makeWorkerAgent = workerFromBackend(opts.backend, opts.deliverable) + const makeLeaf = workerFromBackend(options.backend, options.deliverable) + const securityPolicy = options.profileSecurity ?? DEFAULT_AUTHORED_PROFILE_SECURITY_POLICY + + const makeRecursiveWorkerFor = ( + parent: AgentProfile, + parentIdentity: NodeExecutionIdentity, + depth: number, + parentOwnerId: string, + ): MakeWorkerAgent => { + const makeRecursiveWorker: MakeWorkerAgent = (authoredProfile, spawnContext) => { + if (!spawnContext) { + throw new ValidationError('supervise: backend-derived workers require spawn context') + } + const input = freezeDetachedProfile(authoredProfile) + const authorizationInput = Object.freeze({ + profile: input, + parent, + parentIdentity, + parentNodeId: spawnContext.parentNodeId, + assignmentId: spawnContext.assignmentId, + task: spawnContext.task, + budget: spawnContext.budget, + label: spawnContext.label, + ...(spawnContext.key !== undefined ? { key: spawnContext.key } : {}), + depth, + }) + const decision = options.authorizeSpawn + ? freezeDetached(options.authorizeSpawn(authorizationInput)) + : Object.freeze({ + profile: input, + ...(spawnContext.execution ? { execution: spawnContext.execution } : {}), + }) + if (typeof decision !== 'object' || decision === null || Array.isArray(decision)) { + throw new ValidationError('supervise: authorizeSpawn must return an AuthorizedSpawn') + } + const authorized = freezeDetachedProfile(decision.profile) + const childExecution = canonicalExecution( + authorized, + spawnContext.task, + decision.execution, + `supervise spawn ${JSON.stringify(spawnContext.label)}`, + ) + const authorizedContext = Object.freeze({ + ...spawnContext, + ...(childExecution.ref ? { execution: childExecution.ref } : {}), + }) + const security = validateAgentProfileSecurity(authorized, securityPolicy) + if (!security.ok) { + const details = security.issues + .filter((issue) => issue.level === 'error') + .map((issue) => `${issue.code}${issue.path ? ` at ${issue.path}` : ''}`) + .join(', ') + throw new ValidationError(`supervise: spawned AgentProfile refused: ${details}`) + } + assertProfileModelsAllowed(authorized, options.allowedModels) + const driverInput = { profile: authorized, parent, depth } + const isDriver = options.isDriverProfile + ? options.isDriverProfile(driverInput) + : authorized.metadata?.role === 'driver' + if (!isDriver) { + return makeLeaf( + authorized, + Object.freeze({ + ...authorizedContext, + assignmentId: workerAssignmentNamespace( + runNamespace, + parentOwnerId, + spawnContext.assignmentId, + ), + }), + ) + } + const ownerId = childCoordinationOwner( + parentOwnerId, + childExecution.identity, + spawnContext, + depth, + ) + const nestedDriveHarness = driveHarnessForOwner(ownerId) + if (isExternalSupervisor(authorized) && !nestedDriveHarness) { + throw new ValidationError( + `supervise: authored external supervisor profile.harness=${JSON.stringify(authorized.harness)} requires a local bridge driverBackend or an explicit driveHarness with reachable coordination transport`, + ) + } + assertProfileContract( + authorized, + isExternalSupervisor(authorized) + ? (driverMaterialization as ProfileMaterializationContract) + : promptModelProfileMaterialization, + `supervise driver ${JSON.stringify(spawnContext.label)}`, + ) + + const childFactory = makeRecursiveWorkerFor( + authorized, + childExecution.identity, + depth + 1, + ownerId, + ) + const nestedPerWorker = defaultPerWorker(spawnContext.budget) + const authorizeNestedMessage = authorizeDownFor(authorized, depth + 1) + const nested = supervisorAgent(authorized, { + blobs, + makeWorkerAgent: childFactory, + ...(authorizeNestedMessage ? { authorizeDownMessage: authorizeNestedMessage } : {}), + perWorker: nestedPerWorker, + ...(options.router ? { router: options.router } : {}), + ...(nestedDriveHarness ? { driveHarness: nestedDriveHarness } : {}), + nodeContext: { + runId, + runNamespace, + ownerId, + depth, + identity: childExecution.identity, + assignmentId: spawnContext.assignmentId, + }, + ...(options.resolveSupervisorTools + ? { resolveSupervisorTools: options.resolveSupervisorTools } + : {}), + ...(observeNodeEvent ? { observeNodeEvent, replaySettlements: true } : {}), + ...(options.analysts ? { analysts: options.analysts } : {}), + ...(options.analyzeOnSettle ? { analyzeOnSettle: options.analyzeOnSettle } : {}), + ...(options.watchWorkers ? { watchWorkers: options.watchWorkers } : {}), + ...(options.stallAfterMs !== undefined ? { stallAfterMs: options.stallAfterMs } : {}), + ...(options.stopRule ? { stopRule: options.stopRule } : {}), + ...(options.onProgressStop ? { onProgressStop: options.onProgressStop } : {}), + ...(options.maxTurns !== undefined ? { maxTurns: options.maxTurns } : {}), + ...(options.compaction ? { compaction: options.compaction } : {}), + ...(log + ? { + onEvent: (_event, record) => log.append(runId, record, ownerId), + loadPriorCoordination: () => log.load(runId, ownerId), + } + : {}), + ...(options.finalizer ? { finalizer: options.finalizer } : {}), + }) + return driverChild(authorized, nested, journal, childExecution.ref) + } + return makeRecursiveWorker + } + + makeWorkerAgent = makeRecursiveWorkerFor( + canonicalProfile, + rootExecution.identity, + 1, + rootOwnerId, + ) } const workerFactory = makeWorkerAgent - const runId = opts.runId ?? 'supervise' - const log = ctx.coordinationLog - const now = opts.now ?? Date.now - // Every configuration fault above throws SYNCHRONOUSLY — a caller that guards with // `expect(() => supervise(...)).toThrow` still sees the throw, and no compute starts. Only the // durable coordination replay needs to await, so the run begins inside this closure. const start = async () => { - // The durable coordination side-log (file contexts only): replay the prior process's questions - // and findings into the driver, and append this process's as they publish — so a resumed run - // keeps the coordination context the spawn journal does not record. - const priorCoordination = log ? await log.load(runId) : undefined + // The durable coordination side-log (file contexts only) loads prior questions, findings, and + // authorized instruction receipts, then appends this process's evidence as it publishes. The + // router arm receives all three in its resume brief; the external arm seeds prior questions and + // leaves the other evidence in the log. No prior instruction is auto-delivered. + const priorCoordination = log ? await log.load(runId, rootOwnerId) : undefined - const agent = supervisorAgent(profile, { + const authorizeRootMessage = authorizeDownFor(canonicalProfile, 1) + const agent = supervisorAgent(canonicalProfile, { blobs, makeWorkerAgent: workerFactory, + ...(authorizeRootMessage ? { authorizeDownMessage: authorizeRootMessage } : {}), perWorker, - ...(log ? { onEvent: (ev) => log.append(runId, ev, new Date(now()).toISOString()) } : {}), + ...(log + ? { + onEvent: (_event, record) => log.append(runId, record, rootOwnerId), + } + : {}), ...(priorCoordination && - (priorCoordination.questions.length > 0 || priorCoordination.findings.length > 0) + (priorCoordination.questions.length > 0 || + priorCoordination.findings.length > 0 || + priorCoordination.continuations.length > 0 || + priorCoordination.deliveryEvidence.length > 0) ? { priorCoordination } : {}), - ...(opts.finalizer ? { finalizer: opts.finalizer } : {}), - ...(opts.maxLiveWorkers !== undefined ? { maxLiveWorkers: opts.maxLiveWorkers } : {}), - ...(opts.router ? { router: opts.router } : {}), - ...(opts.brain ? { brain: opts.brain } : {}), - ...(opts.driveHarness ? { driveHarness: opts.driveHarness } : {}), - ...(opts.extraTools ? { extraTools: opts.extraTools } : {}), - ...(opts.executeExtraTool ? { executeExtraTool: opts.executeExtraTool } : {}), - ...(opts.analysts ? { analysts: opts.analysts } : {}), - ...(opts.analyzeOnSettle ? { analyzeOnSettle: opts.analyzeOnSettle } : {}), - ...(opts.watchWorkers ? { watchWorkers: opts.watchWorkers } : {}), - ...(opts.stallAfterMs !== undefined ? { stallAfterMs: opts.stallAfterMs } : {}), - ...(opts.stopRule ? { stopRule: opts.stopRule } : {}), - ...(opts.onProgressStop ? { onProgressStop: opts.onProgressStop } : {}), - ...(opts.maxTurns !== undefined ? { maxTurns: opts.maxTurns } : {}), - ...(opts.compaction ? { compaction: opts.compaction } : {}), + ...(options.finalizer ? { finalizer: options.finalizer } : {}), + ...(options.router ? { router: options.router } : {}), + ...(options.brain ? { brain: options.brain } : {}), + ...(rootDriveHarness ? { driveHarness: rootDriveHarness } : {}), + nodeContext: { + runId, + runNamespace, + ownerId: rootOwnerId, + depth: 0, + identity: rootExecution.identity, + }, + ...(options.resolveSupervisorTools + ? { resolveSupervisorTools: options.resolveSupervisorTools } + : {}), + ...(observeNodeEvent ? { observeNodeEvent, replaySettlements: true } : {}), + ...(options.extraTools ? { extraTools: options.extraTools } : {}), + ...(options.executeExtraTool ? { executeExtraTool: options.executeExtraTool } : {}), + ...(options.analysts ? { analysts: options.analysts } : {}), + ...(options.analyzeOnSettle ? { analyzeOnSettle: options.analyzeOnSettle } : {}), + ...(options.watchWorkers ? { watchWorkers: options.watchWorkers } : {}), + ...(options.stallAfterMs !== undefined ? { stallAfterMs: options.stallAfterMs } : {}), + ...(options.stopRule ? { stopRule: options.stopRule } : {}), + ...(options.onProgressStop ? { onProgressStop: options.onProgressStop } : {}), + ...(options.maxTurns !== undefined ? { maxTurns: options.maxTurns } : {}), + ...(options.compaction ? { compaction: options.compaction } : {}), }) - return createSupervisor().run(agent, task, { - budget: opts.budget, + return createSupervisor().run(agent, canonicalTask, { + budget: options.budget, runId, - journal: opts.journal ?? ctx.journal, + journal, blobs, executors: ctx.executors, - maxDepth: opts.maxDepth ?? 8, - ...(opts.probes ? { probes: opts.probes } : {}), + rootIdentity: rootExecution.identity, + ...(rootOwnerRuntime === undefined + ? {} + : { + rootMaterialization: { + runtime: rootOwnerRuntime, + declaration: 'deferred' as const, + authoredProfile: canonicalProfile, + }, + }), + maxDepth: options.maxDepth ?? 8, + ...(options.maxLiveWorkers !== undefined ? { maxLiveWorkers: options.maxLiveWorkers } : {}), + ...(options.probes ? { probes: options.probes } : {}), ...(ctx.resume === true ? { resume: true } : {}), - ...(opts.now ? { now: opts.now } : {}), + ...(options.now ? { now: options.now } : {}), }) } diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index ba2e5418..83ae4a44 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -4,31 +4,40 @@ * no hand-built brain. The supervisor stops being special — it's one profile, materialized by the * same resolution rule as every other agent. * - * - `harness` null/undefined → the in-process router tool-loop: `driverAgent` over the + * - `harness` omitted or `cli-base` → the in-process router tool-loop: `driverAgent` over the * canonical `ToolLoopChat`, built by `routerBrain` from the profile's model + the router seam. - * - `harness` a coding CLI (`claude-code`/`opencode`/`codex`/…) → a SANDBOXED harness drives the + * - `harness` a coding CLI (`claude-code`/`opencode`/`codex`/…) → an EXTERNAL harness drives the * coordination verbs: `serveCoordinationMcp` exposes spawn/await/steer/stop over the live scope, - * and the caller's `driveHarness` runs the harness with that MCP mounted. The harness IS the brain. + * and the caller's `driveHarness` runs the harness with that MCP mounted. `supervise()` builds + * this automatically only for a local bridge; a remote sandbox needs an explicit reachable + * relay or tunnel. The harness IS the brain. * * Both arms spawn children through the SAME `makeWorkerAgent` seam and finalize through the SAME * seam (`runFinalizer` over DELIVERED children only — default keep-best, never the driver's own * prose). */ +import type { AgentProfile } from '@tangle-network/agent-interface' import { ValidationError } from '../../errors' +import type { McpToolDescriptor } from '../../mcp/server' import type { AnalystRegistry, + AuthorizeDownMessage, CoordinationEvent, MakeWorkerAgent, WorkerWatchOptions, } from '../../mcp/tools/coordination' +import { coordinationVerbNames } from '../../mcp/tools/coordination' import { type RouterConfig, routerBrain } from '../router-client' import type { ToolLoopChat, ToolLoopCompactionOptions } from '../tool-loop' import { driverAgent } from './coordination-driver' import type { PriorCoordination } from './coordination-log' import { serveCoordinationMcp } from './coordination-mcp' +import type { BusRecord } from './event-bus' import { bestDelivered, runFinalizer, runTree, type SupervisorFinalizer } from './finalizer' +import { attestRuntimeOwnedScopeOwner, runtimeOwnedScopeOwnerRuntime } from './materialization' +import { detachedSnapshot } from './snapshot' import type { StopRule } from './stop-rules' -import type { Agent, Budget, ResultBlobStore, Scope } from './types' +import type { Agent, Budget, NodeExecutionIdentity, ResultBlobStore, Scope } from './types' /** The standing strategy a router-brained supervisor runs with when its profile names no * `systemPrompt`. The brain's competence IS this prompt: without it the brain has the coordination @@ -52,45 +61,93 @@ export const defaultSupervisorPrompt = [ 'as soon as the deliverable is met.', ].join('\n') -/** The supervisor's profile — the subset of an `AgentProfile` that selects + shapes its brain. - * `harness` is the backend-as-data discriminant; `systemPrompt` is the standing instruction. */ -export interface SupervisorProfile { - readonly name?: string - /** null/undefined → router brain (in-process tool-loop); a coding-CLI harness → sandboxed brain. */ - readonly harness?: string | null - /** The router model when the brain is router-driven (falls back to the deps router config). */ - readonly model?: string - /** The standing instructions ("you delegate, you do not solve"). */ - readonly systemPrompt?: string +/** A supervisor is an ordinary, complete `AgentProfile` playing the supervisor role. + * Runtime policy (budget, concurrency, analysts, stop rules) stays in `SupervisorAgentDeps`; it is + * live execution state, not agent identity. An omitted harness or `cli-base` selects the in-process + * router brain; every other harness is materialized by `driveHarness`. */ +export type SupervisorProfile = AgentProfile + +/** Trusted run/node identity Runtime binds to one manager. Model-authored tool arguments cannot + * provide or replace any of these fields. */ +export interface SupervisorNodeContext { + readonly runId: string + /** Stable across a durable restart; unique per in-memory invocation. */ + readonly runNamespace: string + /** Concrete Scope node that owns this manager's coordination stream. */ + readonly nodeId: string + /** Stable identity of this manager's coordination stream. */ + readonly ownerId: string + readonly depth: number + readonly identity: NodeExecutionIdentity + /** Assignment identity within the parent manager; absent only for the root. */ + readonly assignmentId?: string + readonly profile: SupervisorProfile + readonly task: unknown +} + +/** Context known before `Agent.act`; Runtime adds the concrete node, profile, and task. */ +export type SupervisorNodeContextSeed = Omit + +/** One product-owned tool. It reuses the canonical MCP descriptor fields while Runtime supplies + * the trusted node context as a separate argument and binds the result for either transport. */ +export interface SupervisorToolDescriptor extends Omit { + readonly handler: (raw: unknown, context: SupervisorNodeContext) => Promise } -/** How to run a sandboxed harness as the DRIVER, with the coordination verbs mounted — the substrate +/** Product policy for the tools one exact supervisor node may call. Resolved once per node. */ +export type ResolveSupervisorTools = ( + context: SupervisorNodeContext, +) => ReadonlyArray | Promise> + +/** Context-aware observer used internally to bind product transactions to the actual live node. */ +export type ObserveSupervisorNodeEvent = ( + context: SupervisorNodeContext, + event: CoordinationEvent, + record: BusRecord, +) => void | Promise + +/** How to run an external harness as the DRIVER, with the coordination verbs mounted — the substrate * seam the caller supplies (mirrors `makeWorkerAgent` for spawned children). It runs `profile` on - * `task` in its backend (sandbox / cli-bridge) with `coordinationMcpUrl` mounted as an MCP server, + * `task` in its backend (remote sandbox or local CLI bridge) with `coordinationMcpUrl` mounted as an MCP server, * so the harness calls spawn_agent / await_event / stop as native tools over the live scope. */ export type DriveHarness = (args: { readonly profile: SupervisorProfile readonly task: unknown readonly scope: Scope readonly coordinationMcpUrl: string + /** Data-only product tool surface mounted on the coordination MCP. Runtime-owned drivers include + * this in their materialization evidence without persisting executable handlers. */ + readonly coordinationTools: ReadonlyArray> }) => Promise export interface SupervisorAgentDeps { readonly blobs: ResultBlobStore /** Resolve a spawned worker `profile` to a leaf agent — the recursion seam (same for both arms). */ readonly makeWorkerAgent: MakeWorkerAgent + /** Product authorization for every down-leg continuation to a child. */ + readonly authorizeDownMessage?: AuthorizeDownMessage /** Per-child budget reserved from the conserved pool on each spawn. */ readonly perWorker: Budget /** Hard cap on simultaneously-LIVE workers across both arms — `spawn_agent` fails closed once * this many are in flight (a concurrency fence on top of the conserved-pool fence; bounds live * boxes/sandboxes, not total work). Omit/`<= 0` = no cap. */ readonly maxLiveWorkers?: number - /** Router substrate for a router-brained supervisor (`harness` null). The profile's model wins. */ + /** Router substrate for a router-brained supervisor (`harness` omitted or `cli-base`). The + * profile's model wins. */ readonly router?: RouterConfig /** Inject the brain directly (tests / advanced) instead of resolving `routerBrain` from the profile. */ readonly brain?: ToolLoopChat - /** Required for a sandboxed-harness supervisor (`harness` set): runs the harness as the driver. */ + /** Required to run an external-harness supervisor: runs the harness as the driver. */ readonly driveHarness?: DriveHarness + /** Trusted identity for this manager. Required with node-scoped tools or observation. */ + readonly nodeContext?: SupervisorNodeContextSeed + /** Resolve product-owned tools for this exact manager. Static `extraTools` remain a router-only + * compatibility seam and deliberately receive no new recursive authority. */ + readonly resolveSupervisorTools?: ResolveSupervisorTools + /** Awaited product observation, enriched with this manager's actual live node context. */ + readonly observeNodeEvent?: ObserveSupervisorNodeEvent + /** Replay resume-time settlements through `observeNodeEvent` before the manager starts. */ + readonly replaySettlements?: boolean /** WORK tools the supervisor may call DIRECTLY (router arm) — so it can do simple work ITSELF and * only delegate when it needs parallelism. Pair with `executeExtraTool`. */ readonly extraTools?: ReadonlyArray<{ @@ -127,10 +184,18 @@ export interface SupervisorAgentDeps { readonly compaction?: ToolLoopCompactionOptions /** Pass-through subscriber for every coordination bus event (both arms) — the seam a durable * caller hooks its coordination log onto. */ - readonly onEvent?: (event: CoordinationEvent) => void | Promise - /** Questions + findings replayed from a prior process of this run (a durable coordination log). - * Router arm: seeds the question ledger + the resume brief. Sandbox arm: seeds the ledger. */ + readonly onEvent?: ( + event: CoordinationEvent, + record: BusRecord, + ) => void | Promise + /** Questions, findings, and authorized continuation receipts loaded from a prior process. + * Router arm: questions seed the ledger and all evidence enters the resume brief. External arm: + * questions seed the ledger; receipts remain durable evidence and are never auto-delivered. */ readonly priorCoordination?: PriorCoordination + /** Deferred owner-scoped replay for a recursive supervisor. Its stable owner is known while the + * parent authorizes the child, but loading remains asynchronous; Runtime calls this before the + * nested brain can publish or act on coordination state. */ + readonly loadPriorCoordination?: () => Promise /** How the settled ledger becomes the run's output (both arms). Default `bestDelivered` — the * exact keep-best every existing caller had. Always runs under the delivered-only invariant. */ readonly finalizer?: SupervisorFinalizer @@ -141,71 +206,135 @@ export function supervisorAgent( profile: SupervisorProfile, deps: SupervisorAgentDeps, ): Agent { - const name = profile.name ?? 'supervisor' - const systemPrompt = profile.systemPrompt ?? defaultSupervisorPrompt - const harness = profile.harness ?? null + const stableProfile = detachedSnapshot(profile, 'supervisorAgent profile') + const resolveTools = deps.resolveSupervisorTools + const observeNodeEvent = deps.observeNodeEvent + const nodeContextSeed = + deps.nodeContext === undefined + ? undefined + : detachedSnapshot(deps.nodeContext, 'supervisorAgent node context') + if ((resolveTools || observeNodeEvent) && !nodeContextSeed) { + throw new ValidationError( + 'supervisorAgent: nodeContext is required with resolveSupervisorTools or observeNodeEvent', + ) + } + const name = stableProfile.name ?? 'supervisor' + const systemPrompt = [ + stableProfile.prompt?.systemPrompt ?? defaultSupervisorPrompt, + ...(stableProfile.prompt?.instructions ?? []), + ].join('\n') + const harness = + stableProfile.harness === undefined || stableProfile.harness === 'cli-base' + ? null + : stableProfile.harness if (harness !== null && deps.compaction) { throw new ValidationError( - 'supervisorAgent: compaction is only supported for router-brained supervisors (profile.harness null)', + 'supervisorAgent: compaction is only supported for router-brained supervisors (profile.harness omitted or cli-base)', ) } if (harness === null) { // ROUTER arm: the in-process tool-loop. `routerBrain` is now an internal detail — the caller // passes a profile, not a hand-built brain (a test may still inject `deps.brain`). - const brain = deps.brain ?? routerBrainFromProfile(profile, deps) - return driverAgent({ + const brain = deps.brain ?? routerBrainFromProfile(stableProfile, deps) + const build = ( + priorCoordination?: PriorCoordination, + nodeTools?: ReadonlyArray, + onEvent?: SupervisorAgentDeps['onEvent'], + ) => + driverAgent({ + name, + brain, + blobs: deps.blobs, + makeWorkerAgent: deps.makeWorkerAgent, + ...(deps.authorizeDownMessage ? { authorizeDownMessage: deps.authorizeDownMessage } : {}), + perWorker: deps.perWorker, + systemPrompt, + ...(nodeTools?.length ? { nodeTools } : {}), + ...(deps.maxLiveWorkers !== undefined ? { maxLiveWorkers: deps.maxLiveWorkers } : {}), + ...(deps.extraTools ? { extraTools: deps.extraTools } : {}), + ...(deps.executeExtraTool ? { executeExtraTool: deps.executeExtraTool } : {}), + ...(deps.analysts ? { analysts: deps.analysts } : {}), + ...(deps.analyzeOnSettle ? { analyzeOnSettle: deps.analyzeOnSettle } : {}), + ...(deps.watchWorkers ? { watchWorkers: deps.watchWorkers } : {}), + ...(deps.stallAfterMs !== undefined ? { stallAfterMs: deps.stallAfterMs } : {}), + ...(deps.stopRule ? { stopRule: deps.stopRule } : {}), + ...(deps.onProgressStop ? { onProgressStop: deps.onProgressStop } : {}), + ...(deps.maxTurns !== undefined ? { maxTurns: deps.maxTurns } : {}), + ...(deps.compaction ? { compaction: deps.compaction } : {}), + ...(onEvent ? { onEvent } : {}), + ...(deps.replaySettlements ? { replaySettlements: true } : {}), + ...(priorCoordination ? { priorCoordination } : {}), + ...(deps.finalizer ? { finalizer: deps.finalizer } : {}), + }) + if (!deps.loadPriorCoordination && !resolveTools && !observeNodeEvent) { + return build(deps.priorCoordination, undefined, deps.onEvent) + } + return { name, - brain, - blobs: deps.blobs, - makeWorkerAgent: deps.makeWorkerAgent, - perWorker: deps.perWorker, - systemPrompt, - ...(deps.maxLiveWorkers !== undefined ? { maxLiveWorkers: deps.maxLiveWorkers } : {}), - ...(deps.extraTools ? { extraTools: deps.extraTools } : {}), - ...(deps.executeExtraTool ? { executeExtraTool: deps.executeExtraTool } : {}), - ...(deps.analysts ? { analysts: deps.analysts } : {}), - ...(deps.analyzeOnSettle ? { analyzeOnSettle: deps.analyzeOnSettle } : {}), - ...(deps.watchWorkers ? { watchWorkers: deps.watchWorkers } : {}), - ...(deps.stallAfterMs !== undefined ? { stallAfterMs: deps.stallAfterMs } : {}), - ...(deps.stopRule ? { stopRule: deps.stopRule } : {}), - ...(deps.onProgressStop ? { onProgressStop: deps.onProgressStop } : {}), - ...(deps.maxTurns !== undefined ? { maxTurns: deps.maxTurns } : {}), - ...(deps.compaction ? { compaction: deps.compaction } : {}), - ...(deps.onEvent ? { onEvent: deps.onEvent } : {}), - ...(deps.priorCoordination ? { priorCoordination: deps.priorCoordination } : {}), - ...(deps.finalizer ? { finalizer: deps.finalizer } : {}), - }) + async act(task, scope) { + const context = nodeContextSeed + ? supervisorNodeContext(nodeContextSeed, stableProfile, task, scope) + : undefined + const priorCoordination = await deps.loadPriorCoordination?.() + const nodeTools = + resolveTools && context ? await bindSupervisorTools(resolveTools, context) : undefined + const onEvent = bindSupervisorNodeObserver(context, observeNodeEvent, deps.onEvent) + return build(priorCoordination, nodeTools, onEvent).act(task, scope) + }, + } } - // SANDBOX arm: a sandboxed harness drives the coordination verbs over the live scope. + // EXTERNAL arm: a caller-driven harness uses the coordination verbs over the live scope. const driveHarness = deps.driveHarness if (!driveHarness) { throw new ValidationError( `supervisorAgent: profile.harness="${harness}" needs deps.driveHarness (how to run the harness with the coordination MCP mounted)`, ) } - return { + const externalAgent: Agent = { name, async act(task, scope) { + const context = nodeContextSeed + ? supervisorNodeContext(nodeContextSeed, stableProfile, task, scope) + : undefined + const priorCoordination = deps.loadPriorCoordination + ? await deps.loadPriorCoordination() + : deps.priorCoordination + const nodeTools = + resolveTools && context ? await bindSupervisorTools(resolveTools, context) : undefined + const onEvent = bindSupervisorNodeObserver(context, observeNodeEvent, deps.onEvent) const mcp = await serveCoordinationMcp({ scope, blobs: deps.blobs, makeWorkerAgent: deps.makeWorkerAgent, + ...(deps.authorizeDownMessage ? { authorizeDownMessage: deps.authorizeDownMessage } : {}), perWorker: deps.perWorker, ...(deps.maxLiveWorkers !== undefined ? { maxLiveWorkers: deps.maxLiveWorkers } : {}), ...(deps.analysts ? { analysts: deps.analysts } : {}), ...(deps.analyzeOnSettle ? { analyzeOnSettle: deps.analyzeOnSettle } : {}), ...(deps.watchWorkers ? { watchWorkers: deps.watchWorkers } : {}), ...(deps.stallAfterMs !== undefined ? { stallAfterMs: deps.stallAfterMs } : {}), - ...(deps.onEvent ? { onEvent: deps.onEvent } : {}), - ...(deps.priorCoordination?.questions.length - ? { priorQuestions: deps.priorCoordination.questions } + ...(onEvent ? { onEvent } : {}), + ...(deps.replaySettlements ? { replaySettlements: true } : {}), + ...(priorCoordination?.questions.length + ? { priorQuestions: priorCoordination.questions } : {}), + ...(nodeTools?.length ? { nodeTools } : {}), }) try { - await driveHarness({ profile, task, scope, coordinationMcpUrl: mcp.url }) + await driveHarness({ + profile: stableProfile, + task, + scope, + coordinationMcpUrl: mcp.url, + coordinationTools: (nodeTools ?? []).map(({ name, description, inputSchema }) => ({ + name, + description, + inputSchema, + })), + }) // Drain settled-but-unpulled children first — a gate-verified delivery the harness never // awaited must still reach the finalize ledger. await mcp.drainResolved() @@ -222,6 +351,92 @@ export function supervisorAgent( } }, } + const runtime = runtimeOwnedScopeOwnerRuntime(driveHarness) + return runtime === undefined + ? externalAgent + : attestRuntimeOwnedScopeOwner(externalAgent, runtime) +} + +function supervisorNodeContext( + seed: SupervisorNodeContextSeed, + profile: SupervisorProfile, + task: unknown, + scope: Scope, +): SupervisorNodeContext { + return detachedSnapshot( + { ...seed, nodeId: scope.view.root, profile, task }, + 'supervisorAgent trusted node context', + ) +} + +async function bindSupervisorTools( + resolveTools: ResolveSupervisorTools, + context: SupervisorNodeContext, +): Promise> { + const resolved = await resolveTools(context) + if (!Array.isArray(resolved)) { + throw new ValidationError('supervisorAgent: resolveSupervisorTools must return an array') + } + const names = new Set(coordinationVerbNames) + return Object.freeze( + resolved.map((rawTool, index) => { + if (typeof rawTool !== 'object' || rawTool === null || Array.isArray(rawTool)) { + throw new ValidationError( + `supervisorAgent: resolved tool at index ${index} must be a descriptor`, + ) + } + const { name, description, inputSchema, handler } = rawTool + if (typeof name !== 'string' || name.length === 0) { + throw new ValidationError( + `supervisorAgent: resolved tool at index ${index} needs a non-empty name`, + ) + } + if (names.has(name)) { + throw new ValidationError( + `supervisorAgent: resolved tool "${name}" collides with a coordination verb or another resolved tool`, + ) + } + names.add(name) + if (typeof description !== 'string' || description.length === 0) { + throw new ValidationError(`supervisorAgent: resolved tool "${name}" needs a description`) + } + if (typeof inputSchema !== 'object' || inputSchema === null || Array.isArray(inputSchema)) { + throw new ValidationError(`supervisorAgent: resolved tool "${name}" needs an inputSchema`) + } + if (typeof handler !== 'function') { + throw new ValidationError(`supervisorAgent: resolved tool "${name}" needs a handler`) + } + const descriptor = detachedSnapshot( + { name, description, inputSchema }, + `supervisorAgent resolved tool ${JSON.stringify(name)}`, + ) + return Object.freeze({ + ...descriptor, + handler: (raw: unknown) => + handler( + detachedSnapshot(raw, `supervisorAgent tool ${JSON.stringify(name)} input`), + context, + ), + }) + }), + ) +} + +function bindSupervisorNodeObserver( + context: SupervisorNodeContext | undefined, + observeNodeEvent: ObserveSupervisorNodeEvent | undefined, + onEvent: SupervisorAgentDeps['onEvent'], +): SupervisorAgentDeps['onEvent'] { + if (!observeNodeEvent && !onEvent) return undefined + return async (event, record) => { + if (observeNodeEvent) { + if (!context) { + throw new ValidationError('supervisorAgent: observeNodeEvent has no trusted node context') + } + await observeNodeEvent(context, event, record) + } + await onEvent?.(event, record) + } } function routerBrainFromProfile( @@ -230,8 +445,8 @@ function routerBrainFromProfile( ): ToolLoopChat { if (!deps.router) { throw new ValidationError( - 'supervisorAgent: a router-brained supervisor (harness null) needs deps.router (or deps.brain)', + 'supervisorAgent: a router-brained supervisor (harness omitted or cli-base) needs deps.router (or deps.brain)', ) } - return routerBrain({ ...deps.router, model: profile.model ?? deps.router.model }) + return routerBrain({ ...deps.router, model: profile.model?.default ?? deps.router.model }) } diff --git a/src/runtime/supervise/supervisor.ts b/src/runtime/supervise/supervisor.ts index 795eee30..3129df57 100644 --- a/src/runtime/supervise/supervisor.ts +++ b/src/runtime/supervise/supervisor.ts @@ -34,6 +34,7 @@ * @experimental */ +import { sha256DigestSchema } from '@tangle-network/agent-interface' import { contentAddress, materializeTreeView, @@ -42,10 +43,22 @@ import { } from '../../durable/spawn-journal' import { RuntimeRunStateError } from '../../errors' import { type BudgetPool, createBudgetPool } from './budget' +import { armDeadlineTimer } from './deadline' import { runTree } from './finalizer' -import { createScope } from './scope' +import { + knownExecutionBindingReceipt, + knownMaterializationReceipt, + newExecutionAttemptId, + unknownExecutionBindingReceipt, + unknownMaterializationReceipt, +} from './materialization' +import { createScope, finalizeScopeOwnerMaterialization } from './scope' +import { detachedSnapshot } from './snapshot' import type { Agent, + Budget, + ExecutionBindingReceipt, + ProfileMaterializationReceipt, ResumedKeyState, RootHandle, RootSignal, @@ -91,14 +104,20 @@ function keyedAssignments( .sort((a, b) => a.seq - b.seq) for (const ev of spawns) { if (ev.key === undefined) continue + if (ev.identity === undefined) { + throw new RuntimeRunStateError( + `supervisor: keyed node '${ev.id}' has no durable execution identity`, + ) + } const s = byId.get(ev.id) keys.set( ev.key, s === undefined - ? { id: ev.id, label: ev.label, state: 'in-doubt' } + ? { id: ev.id, label: ev.label, identity: ev.identity, state: 'in-doubt' } : { id: ev.id, label: ev.label, + identity: ev.identity, state: s.kind === 'done' ? 'completed' : 'down', settled: s, }, @@ -107,6 +126,186 @@ function keyedAssignments( return keys } +type SpawnedEvent = Extract + +/** A resumed process may continue only the exact root contract first recorded for this run id. */ +function assertResumeContract(events: SpawnEvent[], opts: SupervisorOpts): SpawnedEvent { + const roots = events.filter( + (event): event is SpawnedEvent => + event.kind === 'spawned' && event.id === opts.runId && event.parent === undefined, + ) + if (roots.length !== 1) { + throw new RuntimeRunStateError( + `supervisor: resumed run '${opts.runId}' must contain exactly one root identity event; found ${roots.length}`, + ) + } + const recorded = roots[0]! + if (contentAddress(recorded.budget) !== contentAddress(opts.budget)) { + throw new RuntimeRunStateError( + `supervisor: resume budget mismatch for run '${opts.runId}'; use a new runId to change limits`, + ) + } + if (!sameOptionalIdentity(recorded.identity, opts.rootIdentity)) { + throw new RuntimeRunStateError( + `supervisor: resume identity mismatch for run '${opts.runId}'; task, profile, candidate, and correlation must match`, + ) + } + const receipts = events.filter( + (event): event is Extract => + event.kind === 'materialized' && event.id === opts.runId, + ) + const expectedReceipt = rootMaterializationReceipt(opts) + if (expectedReceipt === undefined) { + if (receipts.length > 1) { + throw new RuntimeRunStateError( + `supervisor: resumed run '${opts.runId}' contains duplicate root materialization evidence`, + ) + } + } else if ( + receipts.length !== 1 || + contentAddress(receipts[0]?.receipt) !== contentAddress(expectedReceipt) + ) { + throw new RuntimeRunStateError( + `supervisor: resume materialization mismatch for run '${opts.runId}'; backend, model, execution identity, and plan must match`, + ) + } + return recorded +} + +/** Mint root evidence at the trusted composition boundary. A generic in-process Agent root has no + * executor declaration and remains explicitly unknown rather than inheriting fabricated details. */ +function rootMaterializationReceipt( + opts: SupervisorOpts, +): ProfileMaterializationReceipt | undefined { + const declaration = opts.rootMaterialization + if (declaration === undefined) { + return unknownMaterializationReceipt({ + ...(opts.rootIdentity?.profileDigest === undefined + ? {} + : { authoredProfileDigest: opts.rootIdentity.profileDigest }), + runtime: 'inline', + reason: 'root-agent-did-not-report', + }) + } + if (declaration.declaration === 'deferred') return undefined + if (opts.rootIdentity?.profileDigest === undefined) { + throw new RuntimeRunStateError( + `supervisor: run '${opts.runId}' cannot record known root materialization without an exact profileDigest`, + ) + } + try { + return knownMaterializationReceipt({ + authoredProfileDigest: opts.rootIdentity.profileDigest, + runtime: declaration.runtime, + declaration: declaration.declaration, + }) + } catch (error) { + throw new RuntimeRunStateError( + `supervisor: run '${opts.runId}' has invalid root materialization evidence`, + { cause: error }, + ) + } +} + +function rootExecutionBindingReceipt( + opts: SupervisorOpts, + materialization: ProfileMaterializationReceipt, + attemptId: string, +): ExecutionBindingReceipt | undefined { + const root = opts.rootMaterialization + if (root?.declaration === 'deferred') return undefined + if (root === undefined) { + return unknownExecutionBindingReceipt(materialization, attemptId, 'root-agent-did-not-report') + } + try { + return knownExecutionBindingReceipt(materialization, { + attemptId, + ...root.binding, + }) + } catch (error) { + throw new RuntimeRunStateError( + `supervisor: run '${opts.runId}' has invalid root execution binding`, + { cause: error }, + ) + } +} + +function sameOptionalIdentity( + recorded: SpawnedEvent['identity'], + requested: SpawnedEvent['identity'], +): boolean { + if (recorded === undefined || requested === undefined) return recorded === requested + return contentAddress(recorded) === contentAddress(requested) +} + +/** Validate caller-supplied root identity before any journal access. Fresh runs may omit identity, + * but once supplied it is trusted durable evidence and therefore must be complete and canonical; + * resumed runs always require it so the prior root can be matched exactly. */ +function assertRootIdentity(opts: SupervisorOpts): void { + const identity = opts.rootIdentity + if (identity === undefined) { + if (opts.resume !== true) return + throw new RuntimeRunStateError( + `supervisor: resumed run '${opts.runId}' requires an exact rootIdentity with profileDigest and taskDigest`, + ) + } + const unknownFields = Object.keys(identity).filter( + (key) => + key !== 'profileDigest' && + key !== 'taskDigest' && + key !== 'candidateDigest' && + key !== 'correlation', + ) + const correlation = identity.correlation + const validCorrelation = + correlation === undefined || + (typeof correlation === 'object' && + correlation !== null && + !Array.isArray(correlation) && + Object.entries(correlation).every( + ([key, value]) => key.length > 0 && typeof value === 'string' && value.length > 0, + )) + if ( + unknownFields.length > 0 || + !sha256DigestSchema.safeParse(identity.profileDigest).success || + !sha256DigestSchema.safeParse(identity.taskDigest).success || + (identity.candidateDigest !== undefined && + !sha256DigestSchema.safeParse(identity.candidateDigest).success) || + !validCorrelation + ) { + throw new RuntimeRunStateError( + `supervisor: run '${opts.runId}' requires an exact rootIdentity with valid profileDigest, taskDigest, candidateDigest, and correlation`, + ) + } +} + +function rootDeadline(root: SpawnedEvent): number { + const startedAt = Date.parse(root.at) + if (!Number.isFinite(startedAt)) { + throw new RuntimeRunStateError( + `supervisor: root event for '${root.id}' has an invalid timestamp '${root.at}'`, + ) + } + return startedAt + (root.budget.deadlineMs ?? 0) +} + +/** Child reservations whose spawn was durable but whose terminal record never landed. */ +function uncertainSpawnBudgets(events: SpawnEvent[]): Budget[] { + const terminal = new Set( + events + .filter( + (event) => event.kind === 'settled' || event.kind === 'cancelled' || event.kind === 'woken', + ) + .map((event) => event.id), + ) + return events + .filter( + (event): event is SpawnedEvent => + event.kind === 'spawned' && event.parent !== undefined && !terminal.has(event.id), + ) + .map((event) => event.budget) +} + /** Highest `seq` among events matching `pred`, or `-1` when none match (so a resumed scope's * first new ordinal/seq is 0 — the same start a fresh scope uses). */ function maxSeqOf(events: SpawnEvent[], pred: (ev: SpawnEvent) => boolean): number { @@ -134,8 +333,63 @@ export function createSupervisor(): Supervisor { task: Task, opts: SupervisorOpts, ): Promise> { + // Read every caller-owned field once, then detach and freeze all decision data together before + // the first await. Stateful runtime collaborators stay live by reference, but replacing a field + // on the caller's opts object after intake cannot redirect this run. + const { + budget, + rootIdentity, + rootMaterialization, + runId, + journal: journalStore, + blobs: blobStore, + executors: executorRegistry, + probes, + maxDepth, + maxLiveWorkers, + maxRestarts, + withinMs, + resume, + now: suppliedNow, + signal, + hooks, + } = opts + const input = detachedSnapshot( + { + task, + options: { + budget, + runId, + ...(rootIdentity === undefined ? {} : { rootIdentity }), + ...(rootMaterialization === undefined ? {} : { rootMaterialization }), + ...(maxDepth === undefined ? {} : { maxDepth }), + ...(maxLiveWorkers === undefined ? {} : { maxLiveWorkers }), + ...(maxRestarts === undefined ? {} : { maxRestarts }), + ...(withinMs === undefined ? {} : { withinMs }), + ...(resume === undefined ? {} : { resume }), + }, + }, + 'supervisor.run', + ) + opts = Object.freeze({ + ...input.options, + journal: journalStore, + blobs: blobStore, + executors: executorRegistry, + ...(probes === undefined ? {} : { probes }), + ...(suppliedNow === undefined ? {} : { now: suppliedNow }), + ...(signal === undefined ? {} : { signal }), + ...(hooks === undefined ? {} : { hooks }), + }) + task = input.task + const rootAct = root.act.bind(root) const now = opts.now ?? Date.now - const pool = createBudgetPool(opts.budget, now) + assertRootIdentity(opts) + const rootAttemptId = newExecutionAttemptId(opts.runId) + // One instant owns both the fresh pool's absolute deadline and its durable root records. Capture + // it before any asynchronous journal work so a slow begin cannot extend the run on restart. + const runStartedAtMs = now() + const runStartedAt = new Date(runStartedAtMs).toISOString() // RESUME-FIRST (opt-in via `opts.resume`): read any prior journal tree for this runId BEFORE // beginning a fresh one. A non-empty tree means a prior run for this runId already committed @@ -146,7 +400,18 @@ export function createSupervisor(): Supervisor { const prior = opts.resume === true ? await opts.journal.loadTree(opts.runId) : undefined const resuming = prior !== undefined && prior.length > 0 let resumeFrom: ResumeFrom | undefined + let pool: BudgetPool if (resuming) { + const rootEvent = assertResumeContract(prior, opts) + const measured = sumMeasuredSpendFromEvents(prior) + const uncertainReservations = uncertainSpawnBudgets(prior) + pool = createBudgetPool(opts.budget, now, { + committed: addSpend(measured.childWork, measured.driverInference), + uncertainReservations, + ...(rootEvent.budget.deadlineMs !== undefined + ? { absoluteDeadlineMs: rootDeadline(rootEvent) } + : {}), + }) // Rehydrate the committed work: the cursor-ordered `Settled[]` (from the blob store) plus the // tree as it stood at the recorded cursor position. The new scope's ordinal/cursor counters // continue past the recorded maxima so a fresh spawn never reuses a journaled `seq`. @@ -171,6 +436,11 @@ export function createSupervisor(): Supervisor { priorSpend: sumSpendFromEvents(prior), } } else { + pool = createBudgetPool(opts.budget, now, { + ...(opts.budget.deadlineMs !== undefined + ? { absoluteDeadlineMs: runStartedAtMs + opts.budget.deadlineMs } + : {}), + }) // Fresh run: begin the tree and journal the root as its own `spawned` node (parent-less, the // spawn-ordinal-0 marker), so a journal-based reader — `trajectoryReport`, `replaySpawnTree`, // `materializeTreeView` — can reconstruct the WHOLE realized tree from a real run, not only @@ -179,27 +449,59 @@ export function createSupervisor(): Supervisor { // invariant. The uniqueness guard skips `spawned` events (only the cursor namespace must be // unique), so sharing ordinal 0 with the first child's spawn is not a collision; replay ignores // `spawned` events for settlement reconstruction, so the replayed `Settled[]` is unchanged. - await opts.journal.beginTree(opts.runId, new Date(now()).toISOString()) + await opts.journal.beginTree(opts.runId, runStartedAt) + const rootReceipt = rootMaterializationReceipt(opts) + const rootRuntime = opts.rootMaterialization?.runtime ?? 'inline' await opts.journal.appendEvent(opts.runId, { kind: 'spawned', id: opts.runId, label: 'root', budget: opts.budget, - runtime: 'inline', + runtime: rootRuntime, + ...(opts.rootIdentity ? { identity: opts.rootIdentity } : {}), seq: 0, - at: new Date(now()).toISOString(), + at: runStartedAt, }) + if (rootReceipt !== undefined) { + await opts.journal.appendEvent(opts.runId, { + kind: 'materialized', + id: opts.runId, + receipt: rootReceipt, + seq: 0, + at: runStartedAt, + }) + } + } + + const stableRootReceipt = resuming + ? prior?.find( + (event): event is Extract => + event.kind === 'materialized' && event.id === opts.runId, + )?.receipt + : rootMaterializationReceipt(opts) + if (stableRootReceipt !== undefined) { + const rootBinding = rootExecutionBindingReceipt(opts, stableRootReceipt, rootAttemptId) + if (rootBinding !== undefined) { + await opts.journal.appendEvent(opts.runId, { + kind: 'execution-bound', + id: opts.runId, + binding: rootBinding, + seq: 0, + at: runStartedAt, + }) + } } // ONE internal controller is the root scope's abort source. Every cascade path // (caller signal, RootHandle.abort, breaker trip, deadline) aborts it; the scope // fans it out to each live child's executor (acquire-aware reap included). const controller = new AbortController() - const cascadeAbort = (reason?: string) => { - if (controller.signal.aborted) return + const cascadeAbort = (reason?: string): boolean => { + if (controller.signal.aborted) return false // Carry the reason on the signal so it chains down to each child's abort signal // (`childAbort.signal.reason`) — the diagnostic the scope's executors observe. controller.abort(reason) + return true } const onCallerAbort = () => cascadeAbort('caller signal aborted') @@ -214,6 +516,10 @@ export function createSupervisor(): Supervisor { // final result can name it. const breaker = createIntensityBreaker(opts, () => cascadeAbort('intensity breaker tripped')) const journal = wrapJournalForBreaker(opts.journal, breaker) + const priorRootMaterialization = prior?.find( + (event): event is Extract => + event.kind === 'materialized' && event.id === opts.runId, + )?.receipt const scope = createScope({ parentId: opts.runId, @@ -225,9 +531,23 @@ export function createSupervisor(): Supervisor { seams: {}, depth: 0, maxDepth: opts.maxDepth ?? defaultMaxDepth, + ...(opts.maxLiveWorkers !== undefined ? { maxLiveWorkers: opts.maxLiveWorkers } : {}), signal: controller.signal, now, hooks: opts.hooks, + ...(opts.rootMaterialization?.declaration === 'deferred' + ? { + ownerMaterialization: { + runtime: opts.rootMaterialization.runtime, + authoredProfile: opts.rootMaterialization.authoredProfile, + attemptId: rootAttemptId, + requiredKnown: true, + ...(priorRootMaterialization === undefined + ? {} + : { prior: priorRootMaterialization }), + }, + } + : {}), ...(opts.probes ? { probes: opts.probes } : {}), ...(resumeFrom ? { resumeFrom } : {}), }) @@ -242,19 +562,45 @@ export function createSupervisor(): Supervisor { attached.bind({ scope: openScope, cascadeAbort, signal: pushRootSignal(cascadeAbort) }) } - let actOutcome: { ok: true; out: Out } | { ok: false; error: unknown } + let deadlineExceeded = false + const rootDeadlineAtMs = pool.readout().deadlineMs + if (rootDeadlineAtMs > 0 && now() >= rootDeadlineAtMs) { + deadlineExceeded = cascadeAbort('root budget deadline exceeded') + } + const clearRootDeadline = + opts.budget.deadlineMs === undefined + ? undefined + : armDeadlineTimer(Math.max(0, rootDeadlineAtMs - now()), () => { + deadlineExceeded = cascadeAbort('root budget deadline exceeded') + }) + let actOutcome: { ok: true; out: Out } | { ok: false; error: unknown } = { + ok: false, + error: new RuntimeRunStateError('supervisor: root execution did not start'), + } + let executionAborted = controller.signal.aborted try { - const out = await root.act(task, scope) + const out = await runAbortable(() => rootAct(task, scope), controller.signal) actOutcome = { ok: true, out } } catch (error) { // act()'s rejection is the PRIMARY error; capture it before the join barrier so a // teardown failure in the barrier can never overwrite it (firstError precedence). actOutcome = { ok: false, error } } finally { + executionAborted = controller.signal.aborted + clearRootDeadline?.() // Join barrier: tear down every still-live child. Generalizes the kernel's // `finally{ Promise.allSettled(destroy) }` — a teardown throw is allSettled'd and // journaled, never re-thrown. - await drainLiveChildren(openScope, controller) + try { + await drainLiveChildren(openScope, controller) + } catch (error) { + if (actOutcome?.ok !== false) actOutcome = { ok: false, error } + } + try { + await finalizeScopeOwnerMaterialization(openScope) + } catch (error) { + if (actOutcome?.ok !== false) actOutcome = { ok: false, error } + } if (opts.signal) opts.signal.removeEventListener('abort', onCallerAbort) if (attached) attached.unbind() } @@ -263,11 +609,14 @@ export function createSupervisor(): Supervisor { // carried in, so `tree` covers the same work `spentTotal` bills for. Identical to `scope.view` // on every run that did not resume. const tree = runTree(scope) + // Success and failure both pass the same conservation check. A child promise is never allowed + // to disappear behind a swallowed cleanup error and leave a reservation open. + pool.assertNoOpenTickets() if (actOutcome.ok) { + if (executionAborted) return noWinner() // Every child has settled (join barrier above); no reservation may remain. A leaked ticket // would silently corrupt the conserved spend total, so fail loud here — on the success path // only, where the act() error precedence does not apply. - pool.assertNoOpenTickets() const out = actOutcome.out // Completion-oracle at the root: a `winner` MUST carry a real `Out`. A driver that ran to // completion but selected nothing (its keep-best finalize found no DELIVERED child) returns @@ -309,7 +658,7 @@ export function createSupervisor(): Supervisor { const { childWork, driverInference } = await spentFromJournal(journal, opts.runId) return { kind: 'no-winner', - reason: classifyNoWinner(controller, pool, opts, breaker), + reason: classifyNoWinner(controller, pool, opts, breaker, deadlineExceeded), tree, downCount: breaker.downCount(), spentTotal: addSpend(childWork, driverInference), @@ -480,10 +829,23 @@ async function drainLiveChildren( // cancelling one would leave the process pinned to a deadline nobody is reading anymore. const view = scope.view const hasLive = view.inFlight > 0 || view.waiting > 0 - if (!hasLive) return + if (!hasLive) { + if (scope.workerCapacity.live > 0) { + throw new RuntimeRunStateError( + `supervisor: cleanup ended with ${scope.workerCapacity.live} executor resource(s) not confirmed destroyed`, + ) + } + return + } // Cascade the abort into every live child's executor before draining. if (!controller.signal.aborted) controller.abort() - await Promise.allSettled([drainCursor(scope)]) + await drainCursor(scope) + const after = scope.view + if (after.inFlight > 0 || after.waiting > 0 || scope.workerCapacity.live > 0) { + throw new RuntimeRunStateError( + `supervisor: cleanup ended with ${after.inFlight} running, ${after.waiting} waiting, and ${scope.workerCapacity.live} executor resource(s) not confirmed destroyed`, + ) + } } async function drainCursor(scope: Scope): Promise { @@ -493,16 +855,69 @@ async function drainCursor(scope: Scope): Promise { } } +/** Race root policy execution against the same signal that stops every child. The losing promise + * remains observed, so a late rejection cannot surface as an unhandled process error. */ +async function runAbortable(act: () => Promise, signal: AbortSignal): Promise { + if (signal.aborted) throw supervisorAbortError(signal) + return await new Promise((resolve, reject) => { + let settled = false + const cleanup = () => signal.removeEventListener('abort', onAbort) + const onAbort = () => { + queueMicrotask(() => { + if (settled) return + settled = true + cleanup() + reject(supervisorAbortError(signal)) + }) + } + signal.addEventListener('abort', onAbort, { once: true }) + let work: Promise + try { + work = Promise.resolve(act()) + } catch (error) { + cleanup() + reject(error) + return + } + work.then( + (value) => { + if (settled) return + settled = true + cleanup() + resolve(value) + }, + (error) => { + if (settled) return + settled = true + cleanup() + reject(error) + }, + ) + }) +} + +function supervisorAbortError(signal: AbortSignal): Error { + const reason = signal.reason + const error = new Error( + typeof reason === 'string' && reason.length > 0 ? reason : 'supervisor aborted', + ) + error.name = 'AbortError' + return error +} + function classifyNoWinner( controller: AbortController, pool: BudgetPool, opts: SupervisorOpts, breaker: IntensityBreaker, + deadlineExceeded: boolean, ): NoWinnerReason { // A tripped breaker is the most specific cause (children kept dying), so it outranks - // the generic abort it raised. Then a caller/handle abort. Then the pool. The residual - // bucket is "ran to completion under budget but produced nothing usable". + // the abort it raised. A deadline that won the abort race remains budget exhaustion; + // then come caller/handle abort and other exhausted pool channels. The residual bucket is + // "ran to completion under budget but produced nothing usable". if (breaker.tripped()) return 'all-children-down' + if (deadlineExceeded) return 'budget-exhausted' if (controller.signal.aborted) return 'aborted' if (poolExhausted(pool, opts)) return 'budget-exhausted' return 'all-children-down' @@ -510,6 +925,7 @@ function classifyNoWinner( function poolExhausted(pool: BudgetPool, opts: SupervisorOpts): boolean { const r = pool.readout() + if (r.iterationsLeft <= 0) return true if (r.tokensLeft <= 0) return true if (opts.budget.maxUsd !== undefined && r.usdLeft <= 0) return true if ( @@ -546,6 +962,32 @@ async function spentFromJournal( * `metered` = driver inference (re-homed up the tree, so a single root-tree pass already * includes every nested driver's inference). */ function sumSpendFromEvents(events: SpawnEvent[]): { childWork: Spend; driverInference: Spend } { + const totals = sumMeasuredSpendFromEvents(events) + const rootBudget = events.find( + (event): event is SpawnedEvent => event.kind === 'spawned' && event.parent === undefined, + )?.budget + let remainingRootUsd = Math.max( + 0, + (rootBudget?.maxUsd ?? 0) - totals.childWork.usd - totals.driverInference.usd, + ) + for (const budget of uncertainSpawnBudgets(events)) { + totals.childWork.iterations += budget.maxIterations + // The numeric value is the charged upper bound, not a fabricated measurement. The false flag + // makes that distinction machine-readable in every report. + totals.childWork.tokens.input += budget.maxTokens + totals.childWork.tokensKnown = false + const usdCharge = budget.maxUsd ?? (rootBudget?.maxUsd !== undefined ? remainingRootUsd : 0) + totals.childWork.usd += usdCharge + totals.childWork.usdKnown = false + remainingRootUsd = Math.max(0, remainingRootUsd - usdCharge) + } + return totals +} + +function sumMeasuredSpendFromEvents(events: SpawnEvent[]): { + childWork: Spend + driverInference: Spend +} { const childWork: Spend = { iterations: 0, tokens: { input: 0, output: 0 }, usd: 0, ms: 0 } const driverInference: Spend = { iterations: 0, tokens: { input: 0, output: 0 }, usd: 0, ms: 0 } for (const ev of events) { @@ -560,6 +1002,7 @@ function accumulate(a: Spend, b: Spend): void { a.iterations += b.iterations a.tokens.input += b.tokens.input a.tokens.output += b.tokens.output + if (b.tokensKnown === false) a.tokensKnown = false a.usd += b.usd if (b.usdKnown === false) a.usdKnown = false a.ms += b.ms @@ -571,6 +1014,7 @@ function addSpend(a: Spend, b: Spend): Spend { return { iterations: a.iterations + b.iterations, tokens: { input: a.tokens.input + b.tokens.input, output: a.tokens.output + b.tokens.output }, + ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), usd: a.usd + b.usd, ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), ms: a.ms + b.ms, @@ -581,5 +1025,12 @@ function addSpend(a: Spend, b: Spend): Spend { * Checks every channel `addSpend` sums — including `ms` — so the gate stays consistent with the * total even though the coordination driver currently stamps `ms: 0`. */ function isNonEmptySpend(s: Spend): boolean { - return s.iterations > 0 || s.tokens.input > 0 || s.tokens.output > 0 || s.usd > 0 || s.ms > 0 + return ( + s.iterations > 0 || + s.tokens.input > 0 || + s.tokens.output > 0 || + s.tokensKnown === false || + s.usd > 0 || + s.ms > 0 + ) } diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index f382b129..709d1865 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -26,7 +26,7 @@ */ import type { DefaultVerdict } from '@tangle-network/agent-eval' -import type { AgentProfile } from '@tangle-network/agent-interface' +import type { AgentProfile, Sha256Digest } from '@tangle-network/agent-interface' import type { BackendType } from '@tangle-network/sandbox' import type { RuntimeHooks } from '../../runtime-hooks' import type { LoopTokenUsage } from '../types' @@ -84,16 +84,17 @@ export interface Agent { * Built-in implementations (in `runtime.ts`, NOT variants here): router/inline (a direct * Router/HTTP inference call, no box), sandbox (COMPOSES `runAgentRounds` as a leaf, forwarding * PR #150's optional `lineage` passthrough — does NOT reinvent checkpoint/fork), cli - * (Halo/RLM subprocess; `budgetExempt`, excluded from equal-k by construction). A user's + * (Halo/RLM subprocess; `budgetExempt`, refused by budgeted supervision). A user's * own agent (mastra/agno/raw HTTP/anything) is first-class by implementing this interface. */ export interface Executor { /** Stable runtime tag for traces + the equal-k exemption check. */ readonly runtime: Runtime /** - * When true, this executor's spend is NOT metered against the conserved pool and its - * iterations are excluded from the equal-k assertion (a `cli` subprocess without - * token accounting). Fail-loud everywhere else: a metered executor MUST report usage. + * When true, this executor cannot report the usage a conserved pool would need (for example, a + * subscription CLI with no token receipt). `Executor` can still be used directly, but `Scope` + * refuses it before `execute` so unknown compute can never appear as measured zero in a + * supervised or equal-resource run. A metered executor MUST report usage. */ readonly budgetExempt?: boolean /** @@ -144,6 +145,15 @@ export interface Executor { * driver branched on, its verdict, and the conserved spend. Read once, after settle. */ resultArtifact(): { outRef: string; out: Out; verdict?: DefaultVerdict; spent: Spend } + /** + * Optional accounting split for recursive executors. + * `reported` is the child-work spend written on this node's settlement; `reservation` is the + * whole amount reconciled against this node's parent reservation. + * They differ when a driver owns a nested allocation: its child work and own inference consume + * that allocation together, while the journal keeps those two categories separate. + * Valid after `execute` resolves or throws; ordinary leaf executors omit it. + */ + accounting?(): ExecutorAccounting | undefined /** * A driver-executor's OWN-inference subtree total (rolled up from its nested tree's `metered` * events) — the parent scope journals it as a `metered` event for this node on settle, on BOTH @@ -155,6 +165,13 @@ export interface Executor { metered?(): Spend | undefined } +/** Split used by a recursive executor when journaled child work differs from the full amount + * reconciled against its parent reservation. */ +export interface ExecutorAccounting { + readonly reported: Spend + readonly reservation: Spend +} + /** Terminal artifact of a one-shot `Executor.execute`. */ export interface ExecutorResult { outRef: string @@ -180,11 +197,12 @@ export type Runtime = 'router' | 'inline' | 'sandbox' | 'cli' | (string & {}) // ── Executor resolution (OPEN registry, not a switch) ───────────────────────── /** - * `AgentProfile` does NOT carry a `harness`/backend field — `harness` lives on the - * sandbox SDK's `BackendConfig`, not the portable profile. So an agent is mapped to its - * executor through this MINIMAL wrapper, never by fabricating a field onto `AgentProfile`. + * `AgentProfile.harness` is a portable preference; this wrapper records the executor decision for + * one concrete run. A caller may honor the preference, override it for a comparison cell, or supply + * an executor directly, without changing the profile's behavioral identity. * * Resolution (in `runtime.ts`): + * - `executorFactory` present → BYO: build it after admission with the live context. * - `executor` present → BYO: use it verbatim (a user's own `Executor`). * - `harness === null` → router/inline: a direct Router call, no box. * - `harness` is a `BackendType` → sandbox: compose `runAgentRounds` against `profile` on that backend. @@ -194,10 +212,138 @@ export interface AgentSpec { readonly profile: AgentProfile /** `null` selects router/inline; a `BackendType` selects the sandboxed harness. */ readonly harness: BackendType | null + /** Trusted candidate/campaign attribution supplied by the caller. Profile/task digests are + * computed by Scope from the exact values it executes and cannot be supplied here. */ + readonly execution?: AgentExecutionRef + /** Per-spawn factory carrying caller configuration. Constructed only after admission, with the + * real child signal and nested-scope context. */ + readonly executorFactory?: ExecutorFactory /** Bring-your-own executor: when set, overrides harness-based resolution entirely. */ readonly executor?: Executor } +/** Caller-owned identity beyond the exact profile/task bytes Scope can compute itself. */ +export interface AgentExecutionRef { + readonly candidateDigest?: Sha256Digest + readonly correlation?: Readonly> +} + +/** Durable identity of one realized node. Missing digests mean the input was not canonical JSON. */ +export interface NodeExecutionIdentity extends AgentExecutionRef { + readonly profileDigest?: Sha256Digest + readonly taskDigest?: Sha256Digest +} + +/** A named model carried into an execution, or an explicit reason the exact model is unknowable. */ +export type MaterializedModelIdentity = + | { readonly status: 'known'; readonly id: string } + | { readonly status: 'unknown'; readonly reason: string } + +/** External execution identity that operators can use to join this node to its backend. */ +export interface MaterializedExecutionIdentity { + /** Backend-native identity kind, for example `request`, `session`, `run`, `process`, or `tree`. */ + readonly kind: string + readonly id: string +} + +/** + * Data-only declaration from trusted executor code about the exact sealed plan `execute` uses. + * Scope snapshots this value and computes the durable receipt; callers never provide digests. + */ +export interface ExecutorMaterialization { + /** Complete profile after trusted runtime-owned attachments or backend overlays were applied. */ + readonly effectiveProfile: AgentProfile + /** Concrete backend or harness selected for this run. */ + readonly backend: string + /** Exact selected model, or an explicit unknown reason. */ + readonly model: MaterializedModelIdentity + /** Backend-native session/run/request/process identity. */ + readonly execution: MaterializedExecutionIdentity + /** Named implementation that turns the effective profile into executable backend inputs. */ + readonly materializer: string + /** Finite JSON describing the exact materialization plan. Persisted by digest only. */ + readonly plan: unknown + /** Trusted runtime-only attachments, such as the coordination MCP. Persisted by digest only. */ + readonly platformAttachments?: unknown +} + +/** Volatile execution routing that is true for one attempt but is not profile identity. The full + * binding is hashed and discarded; only the safe structural descriptor is journaled. */ +export interface ExecutorExecutionBinding { + readonly attemptId: string + readonly binding: unknown + readonly descriptor: Readonly> +} + +/** Why exact materialization evidence is unavailable for a node. */ +export type UnknownMaterializationReason = + | 'executor-did-not-report' + | 'invalid-executor-report' + | 'root-agent-did-not-report' + +/** What the kernel can prove about one node's actual execution plan. */ +export type ProfileMaterializationReceipt = + | { + readonly status: 'known' + readonly authoredProfileDigest: Sha256Digest + readonly effectiveProfileDigest: Sha256Digest + readonly materializationPlanDigest: Sha256Digest + readonly platformAttachmentsDigest?: Sha256Digest + readonly runtime: Runtime + readonly backend: string + readonly model: MaterializedModelIdentity + readonly execution: MaterializedExecutionIdentity + readonly materializer: string + } + | { + readonly status: 'unknown' + readonly authoredProfileDigest?: Sha256Digest + readonly runtime: Runtime + readonly reason: UnknownMaterializationReason + } + +/** One attempt's immutable link from a stable materialization plan to its actual transport. */ +export type ExecutionBindingReceipt = + | { + readonly status: 'known' + readonly attemptId: string + readonly materializationReceiptDigest: Sha256Digest + readonly bindingDigest: Sha256Digest + readonly descriptor: Readonly> + } + | { + readonly status: 'unknown' + readonly attemptId: string + readonly materializationReceiptDigest: Sha256Digest + readonly reason: UnknownMaterializationReason + } + +/** Trusted root composition evidence. Generic `Agent.act` roots omit this and remain unknown. */ +export type RootMaterialization = + | { + readonly runtime: Runtime + readonly declaration: ExecutorMaterialization + readonly binding: Omit + } + | { + /** The runtime-owned external adapter will publish the exact declaration after its dynamic + * platform attachment (for example a coordination URL) exists and before paid work starts. */ + readonly runtime: Runtime + readonly declaration: 'deferred' + /** Exact admitted profile used to validate the stable effective identity at publication. */ + readonly authoredProfile: AgentProfile + } + +/** Kernel-owned context for the concrete supervised node a factory is constructing. */ +export interface ExecutorNodeContext { + readonly rootId: NodeId + readonly parentId: NodeId + readonly nodeId: NodeId + /** Kernel-minted identity for this concrete execution attempt. */ + readonly attemptId: string + readonly identity?: NodeExecutionIdentity +} + /** * Builds a fresh `Executor` for one spawn from the resolved spec. Per-spawn (not * shared) so each child owns its own box/abort/teardown lifecycle. A BYO factory lets a @@ -210,6 +356,8 @@ export type ExecutorFactory = (spec: AgentSpec, ctx: ExecutorContext) => Ex * the factory reaching into module globals. */ export interface ExecutorContext { readonly signal: AbortSignal + /** Present when Scope constructs the executor for a supervised node. */ + readonly node?: ExecutorNodeContext /** Opaque seams the registry threads through; a built-in narrows what it needs. */ readonly seams: Readonly> } @@ -224,8 +372,8 @@ export interface ExecutorRegistry { /** Register a factory for a named runtime. Throws on a duplicate name (fail loud). */ register(runtime: Runtime, factory: ExecutorFactory): void /** - * Resolve a spec to a factory. Precedence: a BYO `spec.executor` → a trivial factory - * returning it; else `harness === null` → the `'router'` factory; else a registered + * Resolve a spec to a factory. Precedence: a BYO `spec.executorFactory` → `spec.executor` → + * `harness === null` → the `'router'` factory; else a registered * factory for the harness-derived runtime. Returns a typed outcome — the caller * inspects `succeeded` before `value` (no silent fallback). */ @@ -249,6 +397,9 @@ export interface Budget { export interface Spend { iterations: number tokens: LoopTokenUsage + /** Token accounting is known unless explicitly false. Missing provider usage must never be + * interpreted as zero under a token ceiling. */ + tokensKnown?: boolean /** Dollar accounting is known unless explicitly false. A false value must not be treated as $0 * when enforcing a dollar-denominated comparison or limit. */ usdKnown?: boolean @@ -281,6 +432,9 @@ export type NodeId = string export interface SpawnOpts { readonly budget: Budget readonly label: string + /** Manager-scoped semantic assignment identity. Unlike `key`, this names every spawn, including + * unkeyed siblings, so product traces can join authorization, node, and backend execution. */ + readonly assignmentId?: string readonly restart?: Restart /** Teardown grace handed to the executor when this node is reaped. */ readonly shutdown?: number | 'brutalKill' | 'infinity' @@ -297,9 +451,17 @@ export interface SpawnOpts { readonly key?: string } -/** Fail-closed spawn rejections: an exhausted pool, an exceeded recursion ceiling, or a `key` - * that is still LIVE in this scope (the same assignment may not run twice concurrently). */ -export type SpawnRejection = 'budget-exhausted' | 'depth-exceeded' | 'duplicate-key' +/** Fail-closed spawn rejections: an exhausted pool, an exceeded recursion ceiling, a full + * tree-wide worker allocation, or a `key` that is still LIVE in this scope (the same assignment + * may not run twice concurrently). */ +export type SpawnRejection = + | 'budget-exhausted' + | 'depth-exceeded' + | 'duplicate-key' + | 'invalid-identity' + | 'key-conflict' + | 'max-live-workers' + | 'scope-aborted' /** * What a KEYED spawn resolved to when the key had a prior attempt. Absent on a fresh key (and on @@ -308,8 +470,10 @@ export type SpawnRejection = 'budget-exhausted' | 'depth-exceeded' | 'duplicate- * `'lost'` DID spawn fresh: the prior attempt settled `down` (retried) or was journaled as * started but never settled — the process died with it in flight and the built-in executors * cannot re-attach to a dead process's work, so the result is explicitly in doubt (lost), never - * silently duplicated. An executor that CAN re-attach to a still-running external execution (a - * live sandbox box) extends this union with an adoption state; none of the built-ins can today. + * silently duplicated. On restart, an in-doubt attempt's full declared reservation is charged and + * its telemetry remains unknown; a fresh retry is admitted only from safely remaining capacity. + * An executor that CAN re-attach to a still-running external execution extends this union with an + * adoption state; none of the built-ins can today. */ export type SpawnPrior = | { readonly state: 'completed'; readonly settled: Settled & { kind: 'done' } } @@ -325,6 +489,14 @@ export interface Handle { readonly id: NodeId readonly label: string readonly status: NodeStatus + /** Manager-scoped assignment identity supplied at admission. */ + readonly assignmentId?: string + /** Durable identity of the authorized profile/task/candidate represented by this handle. */ + readonly identity?: NodeExecutionIdentity + /** Stable execution plan once Runtime has committed it. */ + readonly materialization?: ProfileMaterializationReceipt + /** Immutable per-attempt backend bindings committed so far, oldest first. */ + readonly executionBindings?: ReadonlyArray abort(reason?: string): void /** Phantom: binds the handle to the child's output type so `spawn` returns a * `Handle` distinct from a `Handle`. Type-only — never present at runtime. */ @@ -344,6 +516,8 @@ export type Settled = outRef: string verdict?: DefaultVerdict spent: Spend + /** Epoch ms parsed from the durable settlement record when available. */ + settledAt?: number seq: number } | { @@ -353,6 +527,8 @@ export type Settled = /** True = infrastructure failure (excluded from merge `n` / equal-k), not a bad result. */ infra: boolean restartCount: number + /** Epoch ms parsed from the durable settlement/cancellation record when available. */ + settledAt?: number seq: number } @@ -366,14 +542,18 @@ export type Settled = */ export interface Scope { /** - * Spawn a child. Reserves `opts.budget` from the conserved pool atomically; refunds the - * unspent remainder on settle. Returns a typed outcome — fail-closed on an exhausted - * pool, an exceeded depth ceiling, or a still-live duplicate `key` (the caller inspects - * `ok` before `handle`). A KEYED spawn whose key already settled `done` spends nothing: - * it returns the committed result on `prior` instead of re-running (see `SpawnOpts.key`). + * Spawn a child. For a fresh key or an unkeyed spawn, tree-wide worker admission happens before a + * lazy factory is called, so a full worker allocation creates no worker, executor, or reservation. + * Reserves `opts.budget` from the conserved pool atomically; refunds the unspent remainder on + * settle. Returns a typed outcome — fail-closed on an exhausted pool, an exceeded depth ceiling, a + * full worker allocation, or a still-live duplicate `key` (the caller inspects `ok` before + * `handle`). A KEYED spawn whose key already settled `done` invokes the factory only far enough to + * prepare and authorize the exact profile/task identity, then compares that identity with the + * journal. On a match it spends nothing, constructs no executor, reserves no budget, and runs no + * work: it returns the committed result on `prior` (see `SpawnOpts.key`). */ spawn( - agent: Agent, + agent: Agent | (() => Agent), task: unknown, opts: SpawnOpts, ): { ok: true; handle: Handle; prior?: SpawnPrior } | { ok: false; reason: SpawnRejection } @@ -468,11 +648,20 @@ export interface Scope { /** Conserved-pool readouts (post-reservation). */ readonly budget: Readonly<{ tokensLeft: number + tokensKnown: boolean usdLeft: number usdCapped: boolean + usdKnown: boolean + iterationsLeft: number deadlineMs: number reservedTokens: number }> + /** One tree-wide view of simultaneous spawned work. Every nested scope reads the same counter; + * the root agent itself is not a spawned worker. `freeSlots` is `null` when no limit is set. */ + readonly workerCapacity: Readonly<{ + live: number + freeSlots: number | null + }> } /** @@ -512,6 +701,8 @@ export interface ResumedWork { export interface ResumedKeyState { readonly id: NodeId readonly label: string + /** Identity recorded when this key was first admitted. Every reuse must match it exactly. */ + readonly identity?: NodeExecutionIdentity readonly state: 'completed' | 'down' | 'in-doubt' /** The rehydrated settlement; absent exactly when `state` is `'in-doubt'`. */ readonly settled?: Settled @@ -526,6 +717,15 @@ export interface NodeSnapshot { readonly status: NodeStatus readonly runtime: Runtime readonly budget: Budget + /** Manager-scoped assignment identity, including deterministic ids for unkeyed siblings. */ + readonly assignmentId?: string + readonly identity?: NodeExecutionIdentity + /** Kernel-owned execution evidence. `unknown` is distinct from a known zero/empty plan. */ + readonly materialization?: ProfileMaterializationReceipt + /** Immutable attempt bindings, oldest first. A retried/resumed node may have more than one. */ + readonly executionBindings?: ReadonlyArray + /** Epoch ms of the terminal journal record; absent while live or when legacy evidence lacks it. */ + readonly settledAt?: number /** Conserved spend so far for this node. */ readonly spent: Spend /** `outRef` once the node is `done` (the replay/result pointer). */ @@ -557,8 +757,29 @@ export type SpawnEvent = /** The semantic spawn key (`SpawnOpts.key`), when the spawn carried one — what a resumed * run matches to resolve the same assignment to its committed result. */ key?: string + /** Manager-scoped assignment identity used to join unkeyed and keyed work alike. */ + assignmentId?: string budget: Budget runtime: Runtime + /** Exact profile/task digests plus trusted candidate/campaign attribution when available. */ + identity?: NodeExecutionIdentity + seq: number + at: string + } + | { + /** Volatile transport/session binding for exactly one attempt. The full binding is retained + * only by digest; descriptor fields are safe structural labels, never credential-bearing URLs. */ + kind: 'execution-bound' + id: NodeId + binding: ExecutionBindingReceipt + seq: number + at: string + } + | { + /** Trusted runtime transformation from the authorized profile to actual wire bytes. */ + kind: 'materialized' + id: NodeId + receipt: ProfileMaterializationReceipt seq: number at: string } @@ -650,6 +871,11 @@ export interface Supervisor { export interface SupervisorOpts { /** The root conserved-pool ceiling (tokens + usd + iterations + deadline). */ readonly budget: Budget + /** Exact root profile/task identity supplied by the one-call composition surface. */ + readonly rootIdentity?: NodeExecutionIdentity + /** Trusted composition evidence for a root whose `act` drives an external backend. A generic + * root omits it and is durably marked unknown; model-facing Scope never receives this writer. */ + readonly rootMaterialization?: RootMaterialization /** Trace-correlation root + the journal/blob root key. */ readonly runId: NodeId /** Event source — defaults to the in-memory journal in the impl; pass JSONL/FS for durability. */ @@ -664,6 +890,9 @@ export interface SupervisorOpts { readonly probes?: WaitProbeRegistry /** Runtime recursion-depth ceiling (paired with the conserved pool per R3). */ readonly maxDepth?: number + /** Hard tree-wide cap on simultaneously executing spawned workers. The root is excluded; every + * nested driver and leaf shares this one allocation. Omit/`<= 0` leaves worker count uncapped. */ + readonly maxLiveWorkers?: number /** * OTP intensity breaker: more than `maxRestarts` child restarts within `withinMs` * trips the supervisor to `no-winner` rather than restarting forever. diff --git a/tests/helpers/supervisor-resume-child.ts b/tests/helpers/supervisor-resume-child.ts index 2165782d..fadd5916 100644 --- a/tests/helpers/supervisor-resume-child.ts +++ b/tests/helpers/supervisor-resume-child.ts @@ -12,7 +12,7 @@ */ import { appendFileSync } from 'node:fs' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' import { spendFromUsageEvents } from '../../src/runtime/supervise/budget' import { createFileRunContext } from '../../src/runtime/supervise/run-context' import { createSupervisor } from '../../src/runtime/supervise/supervisor' @@ -148,6 +148,10 @@ const root: Agent = { const ctx = createFileRunContext(dir) const result = await createSupervisor().run(root, 'task', { budget: { maxIterations: 50, maxTokens: 100_000 }, + rootIdentity: { + profileDigest: canonicalCandidateDigest({ name: root.name }), + taskDigest: canonicalCandidateDigest('task'), + }, runId, ...ctx, now: () => (phase === '1' ? 1_000 : 2_000), diff --git a/tests/helpers/supervisor-wait-child.ts b/tests/helpers/supervisor-wait-child.ts index 7347cd32..0bc40139 100644 --- a/tests/helpers/supervisor-wait-child.ts +++ b/tests/helpers/supervisor-wait-child.ts @@ -17,6 +17,7 @@ */ import { existsSync } from 'node:fs' +import { canonicalCandidateDigest } from '@tangle-network/agent-interface' import { FileSpawnJournal } from '../../src/durable/spawn-journal' import { createFileRunContext } from '../../src/runtime/supervise/run-context' import { createSupervisor } from '../../src/runtime/supervise/supervisor' @@ -119,6 +120,10 @@ async function killOnceJournaled(): Promise { const ctx = createFileRunContext(dir) const result = await createSupervisor().run(root, 'task', { budget: { maxIterations: 50, maxTokens: 100_000 }, + rootIdentity: { + profileDigest: canonicalCandidateDigest({ name: root.name }), + taskDigest: canonicalCandidateDigest('task'), + }, runId, ...ctx, probes, diff --git a/tests/kernel/coordination-mcp.test.ts b/tests/kernel/coordination-mcp.test.ts index e0496193..292bddc6 100644 --- a/tests/kernel/coordination-mcp.test.ts +++ b/tests/kernel/coordination-mcp.test.ts @@ -104,4 +104,67 @@ describe('coordination MCP over a live Scope — the real keystone (HTTP → MCP expect(names).toContain('spawn_agent') expect(names).toContain('await_event') }) + + it('serves product-owned node tools beside coordination tools over the same HTTP MCP', async () => { + const calls: unknown[] = [] + const scope = {} as Scope + const mcp = await serveCoordinationMcp({ + scope, + blobs: new InMemoryResultBlobStore(), + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker: { maxIterations: 1, maxTokens: 1 }, + nodeTools: [ + { + name: 'lookup_evidence', + description: 'Read product evidence', + inputSchema: { + type: 'object', + properties: { query: { type: 'string' } }, + required: ['query'], + }, + handler: async (raw) => { + calls.push(raw) + return { result: 'trusted evidence' } + }, + }, + ], + }) + try { + const listed = await jsonRpc(mcp.url, 'tools/list', {}) + const names = ((listed.result as { tools?: Array<{ name: string }> })?.tools ?? []).map( + (entry) => entry.name, + ) + expect(names).toContain('spawn_agent') + expect(names).toContain('lookup_evidence') + + const called = await jsonRpc(mcp.url, 'tools/call', { + name: 'lookup_evidence', + arguments: { query: 'claim' }, + }) + expect(called.error).toBeUndefined() + expect(called.result).toMatchObject({ structuredContent: { result: 'trusted evidence' } }) + expect(calls).toEqual([{ query: 'claim' }]) + } finally { + await mcp.close() + } + }) + + it('refuses a product tool that shadows spawn_agent before opening a listener', async () => { + await expect( + serveCoordinationMcp({ + scope: {} as Scope, + blobs: new InMemoryResultBlobStore(), + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker: { maxIterations: 1, maxTokens: 1 }, + nodeTools: [ + { + name: 'spawn_agent', + description: 'must not shadow coordination', + inputSchema: { type: 'object' }, + handler: async () => ({}), + }, + ], + }), + ).rejects.toThrow(/spawn_agent.*shadows/) + }) }) diff --git a/tests/kernel/coordination.test.ts b/tests/kernel/coordination.test.ts index 2c3f2c39..94d12530 100644 --- a/tests/kernel/coordination.test.ts +++ b/tests/kernel/coordination.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { createMcpServer } from '../../src/mcp/server' -import { createCoordinationTools } from '../../src/mcp/tools/coordination' +import { type CoordinationEvent, createCoordinationTools } from '../../src/mcp/tools/coordination' import type { Agent, ResultBlobStore, Scope, Spend } from '../../src/runtime' import { createPushTraceSource, watchTrace } from '../../src/runtime' @@ -16,6 +16,10 @@ function mockScope() { status: 'running' as const, runtime: 'router', budget: { maxIterations: 1, maxTokens: 10 }, + identity: { + profileDigest: `sha256:${'a'.repeat(64)}`, + taskDigest: `sha256:${'b'.repeat(64)}`, + }, spent: zeroSpend(), }, { @@ -76,6 +80,7 @@ describe('coordination tools', () => { // No `maxLiveWorkers` cap ⇒ `freeSlots: null` (uncapped; the conserved pool is the fence). expect(await tool(tb, 'spawn_agent').handler({ profile: {}, task: 'go' })).toEqual({ workerId: 'w0', + assignmentId: 'ordinal:0', live: 1, freeSlots: null, }) @@ -126,14 +131,29 @@ describe('coordination tools', () => { const spawn = () => tool(tb, 'spawn_agent').handler({ profile: {}, task: 'go' }) // `freeSlots` counts down as the cap fills — the reading that tells the driver capacity is // still idle, so it can fill slots instead of opening one worker per turn. - expect(await spawn()).toEqual({ workerId: 'w0', live: 1, freeSlots: 1 }) - expect(await spawn()).toEqual({ workerId: 'w1', live: 2, freeSlots: 0 }) + expect(await spawn()).toEqual({ + workerId: 'w0', + assignmentId: 'ordinal:0', + live: 1, + freeSlots: 1, + }) + expect(await spawn()).toEqual({ + workerId: 'w1', + assignmentId: 'ordinal:1', + live: 2, + freeSlots: 0, + }) // The 2 live workers fill the cap → the 3rd fails closed BEFORE scope.spawn is called. expect(await spawn()).toEqual({ error: 'max-live-workers', live: 2, freeSlots: 0 }) expect(spawns).toHaveLength(2) // A settled worker frees a slot — mark one terminal and the next spawn admits again. live[0]!.status = 'done' - expect(await spawn()).toEqual({ workerId: 'w2', live: 2, freeSlots: 0 }) + expect(await spawn()).toEqual({ + workerId: 'w2', + assignmentId: 'ordinal:2', + live: 2, + freeSlots: 0, + }) // No cap (omitted) → the pool stays the only fence; the same scope admits past the prior cap. const uncapped = createCoordinationTools({ @@ -144,6 +164,7 @@ describe('coordination tools', () => { }) expect(await tool(uncapped, 'spawn_agent').handler({ profile: {}, task: 'go' })).toEqual({ workerId: 'w3', + assignmentId: 'ordinal:0', live: 3, freeSlots: null, }) @@ -210,7 +231,12 @@ describe('coordination tools', () => { tool(tb, 'spawn_agent').handler({ profile: {}, task: 'go', key }) // Key 'a' runs and takes the only slot, then delivers. - expect(await spawnKeyed('a')).toEqual({ workerId: 'w0', live: 1, freeSlots: 0 }) + expect(await spawnKeyed('a')).toEqual({ + workerId: 'w0', + assignmentId: 'key:a', + live: 1, + freeSlots: 0, + }) live[0]!.status = 'done' deliveredKey = 'a' // Drain the settlement the way the driver does — this is what teaches the toolbox that key @@ -219,7 +245,12 @@ describe('coordination tools', () => { await tool(tb, 'await_event').handler({ kinds: ['settled'] }) // A different assignment now occupies the single slot. - expect(await spawnKeyed('b')).toEqual({ workerId: 'w1', live: 1, freeSlots: 0 }) + expect(await spawnKeyed('b')).toEqual({ + workerId: 'w1', + assignmentId: 'key:b', + live: 1, + freeSlots: 0, + }) // Re-asking for the DELIVERED key at the cap must return its committed result, not a refusal. expect(await spawnKeyed('a')).toEqual({ @@ -229,6 +260,7 @@ describe('coordination tools', () => { score: 1, valid: true, outRef: 'blob:a', + spent: zeroSpend(), live: 1, freeSlots: 0, }) @@ -264,7 +296,12 @@ describe('coordination tools', () => { task: 'hard', budget: { maxTokens: 5000, maxUsd: 0.5 }, }), - ).toEqual({ workerId: 'w0', live: 1, freeSlots: null }) + ).toEqual({ + workerId: 'w0', + assignmentId: 'ordinal:0', + live: 1, + freeSlots: null, + }) expect(spawns[0].opts.budget).toEqual({ maxIterations: 2, maxTokens: 5000, maxUsd: 0.5 }) }) @@ -368,6 +405,7 @@ describe('coordination tools', () => { score: 0.83, valid: true, outRef: 'blob:w7', + spent: zeroSpend(), freeSlots: null, }) expect(await tool(tb, 'await_event').handler({ kinds: ['settled'] })).toEqual({ @@ -377,9 +415,9 @@ describe('coordination tools', () => { expect(tb.settled()).toMatchObject([ { id: 'w7', status: 'done', score: 0.83, valid: true, outRef: 'blob:w7' }, ]) - // The ledger stamps WHEN the settlement landed — the resolution a progress-based stop rule - // reads to answer "how long since anything landed?" without inventing a timestamp at read time. - expect(typeof tb.settled()[0]?.settledAt).toBe('number') + // This hand-rolled scope supplied no durable terminal timestamp. Missing stays missing; the + // coordination layer never invents a read-time value that would change on replay. + expect(tb.settled()[0]?.settledAt).toBeUndefined() }) it('await_event bounds the block: { pending, live } while a worker runs, then pulls the settlement once it lands', async () => { @@ -439,7 +477,7 @@ describe('coordination tools', () => { it('blocks stop under failClosed until a parent question is answered', async () => { const { scope } = mockScope() - const emitted: unknown[] = [] + const emitted: CoordinationEvent[] = [] const tb = createCoordinationTools({ scope, blobs, @@ -450,7 +488,7 @@ describe('coordination tools', () => { }) const r = (await tool(tb, 'ask_parent').handler({ - from: 'driver-1', + from: 'w0', level: 'driver', question: 'Which API version should this migration target?', reason: 'worker found two supported versions', @@ -460,27 +498,101 @@ describe('coordination tools', () => { stopped: false, error: 'unresolved-blocking-questions', }) - // The driver-1 asker is not a live worker in the mock scope → the answer reports delivered:false. + // The asker is a live worker, so accepted delivery resolves the blocking question. expect( await tool(tb, 'answer_question').handler({ questionId: r.question.id, answer: 'Target v2.', by: 'user', }), - ).toMatchObject({ question: { status: 'answered' }, delivered: false }) + ).toMatchObject({ question: { status: 'answered' }, delivered: true }) expect(await tool(tb, 'stop').handler({ reason: 'answered and verified' })).toEqual({ stopped: true, }) expect(tb.questions()[0]).toMatchObject({ status: 'answered' }) - // The pass-through trail records BOTH legs: the question up, then the answer routed down. - expect(emitted).toEqual([ + // The exact answer is committed before it is routed down. + expect(emitted).toMatchObject([ { type: 'question', question: expect.objectContaining(r.question) }, + { + type: 'instruction', + instruction: expect.objectContaining({ + kind: 'answer', + toWorker: 'w0', + instruction: 'Target v2.', + questionId: r.question.id, + }), + }, + { + type: 'delivery-attempt', + attempt: expect.objectContaining({ + kind: 'answer', + toWorker: 'w0', + questionId: r.question.id, + }), + }, { type: 'answer', questionId: r.question.id, - down: { toWorker: 'driver-1', instruction: 'Target v2.', delivered: false }, + down: expect.objectContaining({ + toWorker: 'w0', + instruction: 'Target v2.', + delivered: true, + outcome: 'delivered', + }), }, ]) + const receipt = emitted[1] + const attempt = emitted[2] + const outcome = emitted[3] + if ( + receipt?.type !== 'instruction' || + attempt?.type !== 'delivery-attempt' || + outcome?.type !== 'answer' + ) { + throw new Error('expected authorization, attempt, and answer outcome evidence') + } + expect(attempt.attempt.receiptId).toBe(receipt.instruction.receiptId) + expect(outcome.down.receiptId).toBe(receipt.instruction.receiptId) + expect(outcome.down.instructionDigest).toBe(receipt.instruction.instructionDigest) + }) + + it('keeps a blocking question open when its authorized answer cannot reach the worker', async () => { + const { scope } = mockScope() + const emitted: CoordinationEvent[] = [] + const tb = createCoordinationTools({ + scope, + blobs, + makeWorkerAgent, + perWorker: { maxIterations: 1, maxTokens: 10 }, + questionPolicy: 'failClosed', + onEvent: (event) => emitted.push(event), + }) + const raised = (await tool(tb, 'ask_parent').handler({ + from: 'missing-worker', + level: 'worker', + question: 'Which target?', + reason: 'blocked on a choice', + urgency: 'blocks-run', + })) as { question: { id: string } } + + const answer = await tool(tb, 'answer_question').handler({ + questionId: raised.question.id, + answer: 'Target B', + }) + expect(answer).toMatchObject({ + question: { id: raised.question.id, status: (raised.question as { status: string }).status }, + delivered: false, + reason: 'unknown-worker', + }) + expect(tb.questions()).toMatchObject([ + { id: raised.question.id, status: (raised.question as { status: string }).status }, + ]) + expect(await tool(tb, 'stop').handler({ reason: 'cannot claim done' })).toMatchObject({ + stopped: false, + error: 'unresolved-blocking-questions', + }) + const outcome = emitted.find((event) => event.type === 'answer') + expect(outcome?.down).toMatchObject({ delivered: false, outcome: 'unknown-worker' }) }) it('list_analysts surfaces the menu and run_analyst applies a lens to a settled worker', async () => { @@ -558,7 +670,7 @@ describe('coordination tools', () => { it('steer_agent routes down + records in history but is never pulled back', async () => { const { scope, sent } = mockScope() - const emitted: Array<{ type: string }> = [] + const emitted: CoordinationEvent[] = [] const tb = createCoordinationTools({ scope, blobs, @@ -582,15 +694,110 @@ describe('coordination tools', () => { // The forceful steer reached the child inbox (down delivery)... expect(sent).toEqual([{ id: 'w0', msg: { steer: 'do X', interrupt: true } }]) // ...and both attempts were recorded for observability (pass-through + history)... - expect(emitted.map((e) => e.type)).toEqual(['steer', 'steer']) - expect(tb.history().map((r) => r.event.type)).toEqual(['steer', 'steer']) + expect(emitted.map((e) => e.type)).toEqual([ + 'instruction', + 'delivery-attempt', + 'steer', + 'instruction', + 'delivery-attempt', + 'steer', + ]) + expect(tb.history().map((r) => r.event.type)).toEqual([ + 'instruction', + 'delivery-attempt', + 'steer', + 'instruction', + 'delivery-attempt', + 'steer', + ]) + const [ + acceptedReceipt, + acceptedAttempt, + acceptedOutcome, + refusedReceipt, + refusedAttempt, + refusedOutcome, + ] = emitted + if ( + acceptedReceipt?.type !== 'instruction' || + acceptedAttempt?.type !== 'delivery-attempt' || + acceptedOutcome?.type !== 'steer' || + refusedReceipt?.type !== 'instruction' || + refusedAttempt?.type !== 'delivery-attempt' || + refusedOutcome?.type !== 'steer' + ) { + throw new Error('expected two complete delivery evidence chains') + } + expect(acceptedAttempt.attempt.receiptId).toBe(acceptedReceipt.instruction.receiptId) + expect(acceptedOutcome.down).toMatchObject({ + receiptId: acceptedReceipt.instruction.receiptId, + outcome: 'delivered', + delivered: true, + }) + expect(refusedAttempt.attempt.receiptId).toBe(refusedReceipt.instruction.receiptId) + expect(refusedOutcome.down).toMatchObject({ + receiptId: refusedReceipt.instruction.receiptId, + outcome: 'unknown-worker', + delivered: false, + }) // ...but the parent never pulls its own outbound messages back. expect(await tool(tb, 'await_event').handler({})).toEqual({ idle: true, freeSlots: null }) }) + it('authorizes and commits the exact continuation before delivery', async () => { + const { scope } = mockScope() + const steps: string[] = [] + const mutable = scope as unknown as { + send(id: string, message: unknown): boolean + } + mutable.send = (_id, message) => { + steps.push(`send:${JSON.stringify(message)}`) + return true + } + const tb = createCoordinationTools({ + scope, + blobs, + makeWorkerAgent, + perWorker: { maxIterations: 1, maxTokens: 10 }, + authorizeDownMessage(input) { + steps.push(`authorize:${input.instruction}`) + expect(Object.isFrozen(input)).toBe(true) + expect(Object.isFrozen(input.workerIdentity)).toBe(true) + return { instruction: 'AUTHORIZED CONTINUATION' } + }, + onEvent(event) { + if (event.type === 'instruction') steps.push(`commit:${event.instruction.instruction}`) + if (event.type === 'delivery-attempt') steps.push(`attempt:${event.attempt.kind}`) + if (event.type === 'steer') steps.push(`outcome:${event.down.outcome}`) + }, + }) + + await tool(tb, 'steer_agent').handler({ workerId: 'w0', instruction: 'authored text' }) + + expect(steps).toEqual([ + 'authorize:authored text', + 'commit:AUTHORIZED CONTINUATION', + 'attempt:steer', + 'send:{"steer":"AUTHORIZED CONTINUATION","interrupt":false}', + 'outcome:delivered', + ]) + expect(tb.history()[0]?.event).toMatchObject({ + type: 'instruction', + instruction: { + kind: 'steer', + toWorker: 'w0', + instruction: 'AUTHORIZED CONTINUATION', + workerIdentity: { + profileDigest: `sha256:${'a'.repeat(64)}`, + taskDigest: `sha256:${'b'.repeat(64)}`, + }, + }, + }) + }) + it('answer_question routes the answer down to a LIVE worker and surfaces delivered:true', async () => { const { scope, sent } = mockScope() - const emitted: Array<{ type: string }> = [] + const emitted: CoordinationEvent[] = [] const tb = createCoordinationTools({ scope, blobs, @@ -617,8 +824,13 @@ describe('coordination tools', () => { expect(sent).toEqual([ { id: 'w0', msg: { answer: 'path B', questionId: r.question.id, interrupt: true } }, ]) - // ...and both legs are on the trail: question up, answer down. - expect(emitted.map((e) => e.type)).toEqual(['question', 'answer']) + // ...and the committed exact instruction sits between question-up and answer-down. + expect(emitted.map((e) => e.type)).toEqual([ + 'question', + 'instruction', + 'delivery-attempt', + 'answer', + ]) }) it('analyze-on-settle auto-runs lenses and await_event surfaces settled + finding', async () => { @@ -663,6 +875,7 @@ describe('coordination tools', () => { score: 0.1, valid: false, outRef: 'blob:w7', + spent: zeroSpend(), freeSlots: null, }) // The analyze-on-settle finding is now queued; the next pull surfaces it. @@ -679,6 +892,53 @@ describe('coordination tools', () => { expect(emitted).toEqual(['settled', 'finding']) }) + it('retains a settlement when an awaited observer loses its acknowledgement', async () => { + const { scope } = mockScope() + const settlements = [ + { + kind: 'done' as const, + handle: { id: 'w-retry', label: 'w', status: 'done' as const, abort() {} }, + out: { answer: 1 }, + outRef: 'blob:w-retry', + verdict: { valid: true, score: 0.8 }, + spent: zeroSpend(), + seq: 0, + }, + ] + const drainScope = { + ...scope, + next: () => Promise.resolve(settlements.shift() ?? null), + } as typeof scope + const stamps: Array<{ seq: number; at: number }> = [] + let loseAcknowledgement = true + const tb = createCoordinationTools({ + scope: drainScope, + blobs, + makeWorkerAgent, + perWorker: { maxIterations: 1, maxTokens: 10 }, + onEvent(event, record) { + if (event.type !== 'settled') return + stamps.push({ seq: record.seq, at: record.at }) + if (loseAcknowledgement) throw new Error('ack lost after commit') + }, + }) + + await expect(tool(tb, 'await_event').handler({})).rejects.toThrow('ack lost after commit') + expect(tb.settled()).toEqual([]) + expect(tb.history()).toEqual([]) + + loseAcknowledgement = false + await expect(tool(tb, 'await_event').handler({})).resolves.toMatchObject({ + type: 'settled', + settled: 'w-retry', + status: 'done', + }) + expect(stamps).toHaveLength(2) + expect(stamps[1]).toEqual(stamps[0]) + expect(tb.settled()).toHaveLength(1) + expect(tb.history()).toHaveLength(1) + }) + it('await_event with kinds filter waits for a specific message type', async () => { const { scope } = mockScope() const settlements = [ diff --git a/tests/kernel/event-bus.test.ts b/tests/kernel/event-bus.test.ts index f121e16b..774b2502 100644 --- a/tests/kernel/event-bus.test.ts +++ b/tests/kernel/event-bus.test.ts @@ -102,4 +102,28 @@ describe('event bus', () => { await bus.publish({ type: 'finding' }) expect(seen).toEqual(['settled']) }) + + it('keeps an event invisible until awaited subscribers commit and reuses its stamp on retry', async () => { + const bus = createEventBus(fakeClock()) + const event = { type: 'settled', id: 'w1' } as const + const attempts: BusRecord[] = [] + let fail = true + bus.subscribe((record) => { + attempts.push(record) + if (fail) throw new Error('product transaction unavailable') + }) + + await expect(bus.publish(event)).rejects.toThrow('product transaction unavailable') + expect(bus.pending()).toBe(0) + expect(bus.pull()).toBeUndefined() + expect(bus.history()).toEqual([]) + expect(bus.stats()).toEqual({ published: 0, pulled: 0, byKind: {} }) + + fail = false + await expect(bus.publish(event)).resolves.toMatchObject({ seq: 0, at: 1000, event }) + expect(attempts).toHaveLength(2) + expect(attempts[1]).toBe(attempts[0]) + expect(bus.pull()).toBe(event) + expect(bus.stats()).toEqual({ published: 1, pulled: 1, byKind: { settled: 1 } }) + }) }) diff --git a/tests/kernel/supervise-convenience.test.ts b/tests/kernel/supervise-convenience.test.ts index cacedbfb..4174293f 100644 --- a/tests/kernel/supervise-convenience.test.ts +++ b/tests/kernel/supervise-convenience.test.ts @@ -3,8 +3,13 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' +import { InMemorySpawnJournal } from '../../src/durable/spawn-journal' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' -import { supervise, workerFromBackend } from '../../src/runtime/supervise/supervise' +import { + type SuperviseOptions, + supervise, + workerFromBackend, +} from '../../src/runtime/supervise/supervise' import type { Agent, AgentSpec, @@ -40,19 +45,44 @@ function deliveringLeaf(name: string, out: unknown): Agent { } } +function failingLeaf(name: string, reason: string): Agent { + const ex: Executor = { + runtime: 'router', + execute: async () => { + throw new Error(reason) + }, + teardown: () => Promise.resolve({ destroyed: true }), + accounting: () => ({ + reported: { iterations: 0, tokens: { input: 0, output: 0 }, usd: 0, ms: 0 }, + reservation: { iterations: 0, tokens: { input: 0, output: 0 }, usd: 0, ms: 0 }, + }), + resultArtifact: () => { + throw new Error('a failed leaf has no terminal artifact') + }, + } + const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor: ex } + return { name, act: async () => undefined, executorSpec: spec } as Agent & { + executorSpec: AgentSpec + } +} + describe('supervise — the one-call convenience (defaults blobs/perWorker/journal/executors)', () => { it('runs a supervisor to delivery from just profile + task + worker seam + brain + budget', async () => { const brain = scriptedBrain([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'worker' }, task: 'go' } }, + { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, { content: 'done' }, ]) const result = await supervise( - { name: 'root', harness: null, systemPrompt: 'drive the worker' }, + { + name: 'root', + harness: 'cli-base', + prompt: { systemPrompt: 'drive the worker' }, + }, 'solve it', { budget, makeWorkerAgent: () => deliveringLeaf('w', { answer: 42 }), brain }, ) @@ -66,7 +96,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ scriptedBrain([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'worker' }, task: 'go' } }, + { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, @@ -79,7 +109,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ runDir: dir, } - const first = await supervise({ name: 'root', harness: null }, 'solve it', { + const first = await supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { ...opts, brain: script(), }) @@ -97,7 +127,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ // A second `supervise()` against the SAME runDir + runId takes the resume path. Without the // `resume` flag threaded through, this would fail loud in `beginTree` ("already begun at …, // refusing to overwrite") because the wall-clock `at` differs between the two calls. - const second = await supervise({ name: 'root', harness: null }, 'solve it', { + const second = await supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { ...opts, brain: script(), }) @@ -107,7 +137,136 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ } }) - it('workerFromBackend builds a spawnable worker leaf with an executor (no network)', () => { + it('replays one durable settlement with the same event id after the observer commits but loses its acknowledgement', async () => { + const dir = await mkdtemp(join(tmpdir(), 'supervise-observer-retry-')) + try { + const writes = new Set() + let physicalWrites = 0 + let acknowledge = false + let replayCommitted = false + const observations: Array<{ + eventId: string + seq: number + at: number + resumed: boolean + }> = [] + const onCoordinationEvent: NonNullable = async ( + _context, + eventId, + record, + ) => { + if (record.event.type !== 'settled') return + observations.push({ + eventId, + seq: record.seq, + at: record.at, + resumed: record.event.worker?.resumed === true, + }) + if (!writes.has(eventId)) { + writes.add(eventId) + physicalWrites += 1 + } + if (!acknowledge) throw new Error('observer commit succeeded; acknowledgement was lost') + replayCommitted = true + } + const common = { + budget, + makeWorkerAgent: () => deliveringLeaf('worker', { answer: 42 }), + runId: 'observer-retry', + runDir: dir, + onCoordinationEvent, + } + const first = await supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { + ...common, + brain: scriptedBrain([ + { + toolCalls: [ + { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, + ], + }, + { toolCalls: [{ name: 'await_event', arguments: {} }] }, + { content: 'stop after the observer error' }, + ]), + }) + expect(first.kind).toBe('no-winner') + expect(physicalWrites).toBe(1) + expect(observations.length).toBeGreaterThan(0) + expect(new Set(observations.map((entry) => entry.eventId))).toHaveLength(1) + + acknowledge = true + const replayScript = scriptedBrain([ + { toolCalls: [{ name: 'await_event', arguments: { kinds: ['settled'] } }] }, + { content: 'finish from committed work' }, + ]) + const replayBrain: typeof replayScript = async (...args) => { + expect(replayCommitted).toBe(true) + return replayScript(...args) + } + const second = await supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { + ...common, + brain: replayBrain, + }) + + expect(second.kind).toBe('winner') + expect(physicalWrites).toBe(1) + expect(new Set(observations.map((entry) => entry.eventId))).toHaveLength(1) + expect(observations.some((entry) => entry.resumed)).toBe(true) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + it('gives two attempts of one keyed assignment distinct worker and event identities', async () => { + let attempt = 0 + const events: Array<{ eventId: string; workerId: string; assignmentId?: string }> = [] + const result = await supervise({ name: 'root', harness: 'cli-base' }, 'retry once', { + budget, + makeWorkerAgent: () => + attempt++ === 0 + ? failingLeaf('same-worker', 'first attempt failed') + : deliveringLeaf('same-worker', { answer: 42 }), + brain: scriptedBrain([ + { + toolCalls: [ + { + name: 'spawn_agent', + arguments: { profile: { name: 'same-worker' }, task: 'go', key: 'same-assignment' }, + }, + ], + }, + { toolCalls: [{ name: 'await_event', arguments: { kinds: ['settled'] } }] }, + { + toolCalls: [ + { + name: 'spawn_agent', + arguments: { profile: { name: 'same-worker' }, task: 'go', key: 'same-assignment' }, + }, + ], + }, + { toolCalls: [{ name: 'await_event', arguments: { kinds: ['settled'] } }] }, + { content: 'done' }, + ]), + onCoordinationEvent: (_context, eventId, record) => { + if (record.event.type !== 'settled') return + events.push({ + eventId, + workerId: record.event.worker.id, + assignmentId: record.event.worker.assignmentId, + }) + }, + }) + + expect(result.kind).toBe('winner') + expect(events).toHaveLength(2) + expect(events.map((event) => event.assignmentId)).toEqual([ + 'key:same-assignment', + 'key:same-assignment', + ]) + expect(new Set(events.map((event) => event.workerId)).size).toBe(2) + expect(new Set(events.map((event) => event.eventId)).size).toBe(2) + }) + + it('workerFromBackend builds a spawnable worker leaf with a deferred executor factory', () => { const make = workerFromBackend({ backend: 'router-tools', routerBaseUrl: 'http://localhost', @@ -116,18 +275,267 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ } as ExecutorConfig) const w = make({ name: 'w' }) as Agent & { executorSpec: AgentSpec } expect(w.name).toBe('w') - expect(w.executorSpec.executor).toBeDefined() + expect(w.executorSpec.executorFactory).toBeDefined() + expect(w.executorSpec.executor).toBeUndefined() + }) + + it('workerFromBackend captures its reusable backend before callers can redirect it', () => { + const backend: ExecutorConfig = { + backend: 'router', + routerBaseUrl: 'http://router.test', + routerKey: 'key', + model: 'safe-model', + } + const make = workerFromBackend(backend) + const mutableBackend = backend as { backend: string } + mutableBackend.backend = 'cli' + const worker = make({ name: 'worker' }) as Agent & { + executorSpec: AgentSpec + } + const executor = worker.executorSpec.executorFactory?.(worker.executorSpec, { + signal: new AbortController().signal, + seams: {}, + }) + + expect(executor?.runtime).toBe('router') + }) + + it('workerFromBackend rejects post-identity profile overlays and shared execution ids', () => { + const invalid: ExecutorConfig[] = [ + { + backend: 'bridge', + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'model', + agentProfile: { name: 'late-overlay' }, + }, + { + backend: 'bridge', + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'model', + sessionId: 'SHARED', + }, + { + backend: 'cli-worktree', + repoRoot: '/repo', + harness: 'codex', + runId: 'SHARED', + }, + { + backend: 'cli-worktree', + repoRoot: '/repo', + bridge: { + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'model', + sessionId: 'SHARED', + }, + }, + ] + + for (const config of invalid) { + expect(() => workerFromBackend(config)).toThrow(/not allowed|isolated id/) + } + }) + + it('refuses profile behavior a limited backend would silently drop', () => { + const routerWorker = workerFromBackend({ + backend: 'router', + routerBaseUrl: 'http://localhost', + routerKey: 'k', + model: 'm', + }) + expect(() => + routerWorker({ + name: 'rich-worker', + model: { default: 'm', reasoningEffort: 'high' }, + tools: { shell: true }, + }), + ).toThrow(/modelReasoningEffort, tools/) + + const rawCliWorker = workerFromBackend({ backend: 'cli', bin: '/bin/true' }) + expect(() => + rawCliWorker({ name: 'raw-cli', prompt: { systemPrompt: 'This used to be ignored.' } }), + ).toThrow(/systemPrompt/) + + const localWorktreeWorker = workerFromBackend({ + backend: 'cli-worktree', + repoRoot: '/workspace', + harness: 'claude', + }) + expect(() => + localWorktreeWorker({ + name: 'local-worktree', + connections: [{ connectionId: 'github', capabilities: ['issues:read'] }], + }), + ).toThrow(/connections/) + + const bridgedWorktreeWorker = workerFromBackend({ + backend: 'cli-worktree', + repoRoot: '/workspace', + bridge: { bridgeUrl: 'http://localhost', bridgeBearer: 'secret', model: 'm' }, + }) + expect(() => + bridgedWorktreeWorker({ + name: 'bridged-worktree', + connections: [{ connectionId: 'github', capabilities: ['issues:read'] }], + }), + ).not.toThrow() + }) + + it('refuses noncanonical or forged root identity before compute starts', () => { + const makeWorkerAgent = () => deliveringLeaf('w', {}) + expect(() => + supervise( + { name: 'root', harness: 'cli-base' }, + { value: 1n }, + { + budget, + makeWorkerAgent, + brain: scriptedBrain([{ content: 'unused' }]), + }, + ), + ).toThrow(/canonical JSON/) + expect(() => + supervise({ name: 'root', harness: 'cli-base' }, 'task', { + budget, + makeWorkerAgent, + brain: scriptedBrain([{ content: 'unused' }]), + execution: { candidateDigest: 'not-a-digest' as never }, + }), + ).toThrow(/candidateDigest must be a sha256 digest/) + }) + + it('refuses an unsafe authored profile before reserving budget or starting a worker', async () => { + const journal = new InMemorySpawnJournal() + const brain = scriptedBrain([ + { + toolCalls: [ + { + name: 'spawn_agent', + arguments: { + profile: { + name: 'unsafe-worker', + prompt: { systemPrompt: 'run the task' }, + hooks: { beforeTool: [{ command: 'curl https://example.test' }] }, + }, + task: 'go', + }, + }, + ], + }, + { content: 'profile was refused' }, + ]) + const result = await supervise({ name: 'root', harness: 'cli-base' }, 't', { + budget, + backend: { + backend: 'bridge', + bridgeUrl: 'http://127.0.0.1:1', + bridgeBearer: 'unused', + model: 'codex/test', + }, + brain, + journal, + runId: 'unsafe-profile', + }) + + expect(result.kind).toBe('no-winner') + const events = await journal.loadTree('unsafe-profile') + expect(events?.filter((event) => event.kind === 'spawned').map((event) => event.id)).toEqual([ + 'unsafe-profile', + ]) + }) + + it.each([ + { + capability: 'remote MCP', + profile: { + name: 'remote-mcp-worker', + mcp: { + metadata: { transport: 'http' as const, url: 'http://169.254.169.254/latest/meta-data' }, + }, + }, + }, + { + capability: 'hub connection', + profile: { + name: 'connected-worker', + connections: [{ connectionId: 'private-mail', capabilities: ['read'] }], + }, + }, + ])('fails closed on an authored $capability unless the caller grants it', async ({ profile }) => { + const journal = new InMemorySpawnJournal() + const brain = scriptedBrain([ + { + toolCalls: [{ name: 'spawn_agent', arguments: { profile, task: 'go' } }], + }, + { content: 'profile was refused' }, + ]) + + const result = await supervise({ name: 'root', harness: 'cli-base' }, 't', { + budget, + backend: { + backend: 'bridge', + bridgeUrl: 'http://127.0.0.1:1', + bridgeBearer: 'unused', + model: 'codex/test', + }, + brain, + journal, + runId: `unsafe-${profile.name}`, + }) + + expect(result.kind).toBe('no-winner') + const events = await journal.loadTree(`unsafe-${profile.name}`) + expect(events?.filter((event) => event.kind === 'spawned').map((event) => event.id)).toEqual([ + `unsafe-${profile.name}`, + ]) }) it('fails loud with neither backend nor makeWorkerAgent', () => { - expect(() => supervise({ name: 'r', harness: null }, 't', { budget })).toThrow( + expect(() => supervise({ name: 'r', harness: 'cli-base' }, 't', { budget })).toThrow( /backend|makeWorkerAgent/, ) }) + it('refuses spawn authorization with a caller-owned worker factory before anything starts', async () => { + const journal = new InMemorySpawnJournal() + let factoryCalls = 0 + let brainCalls = 0 + let authorizationCalls = 0 + + expect(() => + supervise({ name: 'r', harness: 'cli-base' }, 't', { + budget, + journal, + runId: 'invalid-custom-authority', + makeWorkerAgent: () => { + factoryCalls += 1 + return deliveringLeaf('unused', {}) + }, + brain: async () => { + brainCalls += 1 + return { toolCalls: [], content: 'unused' } + }, + authorizeSpawn(input) { + authorizationCalls += 1 + return { profile: input.profile } + }, + }), + ).toThrow(/authorizeSpawn cannot be combined with caller-owned makeWorkerAgent/) + + expect({ factoryCalls, brainCalls, authorizationCalls }).toEqual({ + factoryCalls: 0, + brainCalls: 0, + authorizationCalls: 0, + }) + expect(await journal.loadTree('invalid-custom-authority')).toBeUndefined() + }) + it('allowedModels rejects a profile model outside the allowed set', () => { expect(() => - supervise({ name: 'r', harness: null, model: 'gpt-4.1' }, 't', { + supervise({ name: 'r', harness: 'cli-base', model: { default: 'gpt-4.1' } }, 't', { budget, makeWorkerAgent: () => deliveringLeaf('w', {}), allowedModels: ['deepseek-v4-flash'], @@ -135,9 +543,86 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ ).toThrow(/gpt-4\.1.*not in the allowed set/) }) + it.each([ + { + field: 'small model', + profile: { model: { default: 'safe', small: 'unsafe-small' } }, + rejected: 'unsafe-small', + }, + { + field: 'subagent model', + profile: { subagents: { critic: { model: 'unsafe-subagent' } } }, + rejected: 'unsafe-subagent', + }, + { + field: 'mode model', + profile: { modes: { review: { model: 'unsafe-mode' } } }, + rejected: 'unsafe-mode', + }, + ])('allowedModels rejects a hidden $field', ({ profile, rejected }) => { + expect(() => + supervise({ name: 'r', harness: 'cli-base', ...profile }, 't', { + budget, + makeWorkerAgent: () => deliveringLeaf('w', {}), + allowedModels: ['safe'], + }), + ).toThrow(new RegExp(`${rejected}.*not in the allowed set`)) + }) + + it('refuses every backend profile overlay before it can bypass authorization', () => { + expect(() => + supervise({ name: 'r', harness: 'cli-base', model: { default: 'safe' } }, 't', { + budget, + backend: { + backend: 'bridge', + bridgeUrl: 'http://127.0.0.1:1', + bridgeBearer: 'unused', + model: 'safe', + agentProfile: { model: { default: 'unsafe-overlay' } }, + }, + allowedModels: ['safe'], + }), + ).toThrow(/backend agentProfile overlays are not allowed/) + }) + + it('refuses a fixed session id on the reusable driver backend', () => { + expect(() => + supervise({ name: 'r', harness: 'codex' }, 't', { + budget, + backend: { + backend: 'bridge', + bridgeUrl: 'http://127.0.0.1:1', + bridgeBearer: 'unused', + model: 'worker-model', + }, + driverBackend: { + backend: 'bridge', + bridgeUrl: 'http://127.0.0.1:1', + bridgeBearer: 'unused', + model: 'driver-model', + sessionId: 'SHARED', + }, + }), + ).toThrow(/driveHarnessFromBackend: fixed sessionId.*isolated id/) + }) + + it('refuses an automatic external supervisor on a backend that cannot receive coordination tools', () => { + expect(() => + supervise({ name: 'r', harness: 'codex' }, 't', { + budget, + backend: { + backend: 'router-tools', + routerBaseUrl: 'http://127.0.0.1:1', + routerKey: 'unused', + model: 'safe', + }, + }), + ).toThrow(/requires a local bridge driverBackend or an explicit driveHarness/) + }) + it('allowedModels rejects a router model outside the allowed set', () => { expect(() => - supervise({ name: 'r', harness: null }, 't', { + supervise({ name: 'r', harness: 'cli-base' }, 't', { budget, makeWorkerAgent: () => deliveringLeaf('w', {}), router: { routerBaseUrl: 'http://localhost', routerKey: 'k', model: 'gpt-4.1' }, @@ -148,7 +633,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ it('allowedModels rejects a backend model outside the allowed set', () => { expect(() => - supervise({ name: 'r', harness: null }, 't', { + supervise({ name: 'r', harness: 'cli-base' }, 't', { budget, backend: { backend: 'router-tools', @@ -162,14 +647,22 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ }) it('allowedModels passes when every configured model is in the set', async () => { - const brain = scriptedBrain([{ content: 'done' }]) const result = await supervise( - { name: 'root', harness: null, model: 'deepseek-v4-flash' }, + { + name: 'root', + harness: 'cli-base', + model: { default: 'deepseek-v4-flash' }, + }, 't', { budget, makeWorkerAgent: () => deliveringLeaf('w', { answer: 1 }), - brain, + router: { + routerBaseUrl: 'http://unused.test', + routerKey: 'test', + model: 'deepseek-v4-flash', + complete: async () => ({ choices: [{ message: { content: 'done' } }] }), + }, allowedModels: ['deepseek-v4-flash'], }, ) @@ -177,12 +670,20 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ }) it('allowedModels unset is unrestricted (any model passes)', async () => { - const brain = scriptedBrain([{ content: 'done' }]) - const result = await supervise({ name: 'root', harness: null, model: 'anything' }, 't', { - budget, - makeWorkerAgent: () => deliveringLeaf('w', { answer: 1 }), - brain, - }) + const result = await supervise( + { name: 'root', harness: 'cli-base', model: { default: 'anything' } }, + 't', + { + budget, + makeWorkerAgent: () => deliveringLeaf('w', { answer: 1 }), + router: { + routerBaseUrl: 'http://unused.test', + routerKey: 'test', + model: 'anything', + complete: async () => ({ choices: [{ message: { content: 'done' } }] }), + }, + }, + ) expect(result.kind).toBeDefined() }) }) diff --git a/tests/kernel/supervise-deadline.test.ts b/tests/kernel/supervise-deadline.test.ts new file mode 100644 index 00000000..c88d97df --- /dev/null +++ b/tests/kernel/supervise-deadline.test.ts @@ -0,0 +1,254 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' +import { createBudgetPool } from '../../src/runtime/supervise/budget' +import { armDeadlineTimer, teardownExecutor } from '../../src/runtime/supervise/deadline' +import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { createScope } from '../../src/runtime/supervise/scope' +import { createSupervisor } from '../../src/runtime/supervise/supervisor' +import type { + Agent, + AgentSpec, + Budget, + Executor, + ExecutorResult, + Scope, + Spend, + SupervisorOpts, +} from '../../src/runtime/supervise/types' + +const zeroSpend: Spend = { + iterations: 0, + tokens: { input: 0, output: 0 }, + usd: 0, + ms: 0, +} + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) + +describe('supervision deadlines', () => { + it('bounds a teardown promise that never acknowledges brutal kill', async () => { + const executor = teardownOnlyExecutor(() => new Promise(() => {})) + + await expect( + teardownExecutor(executor, 'brutalKill', Date.now() + 10, Date.now), + ).rejects.toThrow(/teardown did not acknowledge/) + }) + + it('refuses a teardown receipt that admits the resource survived', async () => { + const executor = teardownOnlyExecutor(async () => ({ destroyed: false })) + + await expect(teardownExecutor(executor, 'brutalKill', undefined, Date.now)).rejects.toThrow( + /destroyed=false/, + ) + }) + + it('chunks delays beyond the Node timer limit instead of firing them after 1 ms', async () => { + vi.useFakeTimers() + vi.setSystemTime(0) + const onDeadline = vi.fn() + const maxTimerDelayMs = 2_147_483_647 + + armDeadlineTimer(maxTimerDelayMs + 500, onDeadline) + await vi.advanceTimersByTimeAsync(maxTimerDelayMs) + expect(onDeadline).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(500) + expect(onDeadline).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + }) + + it('the root deadline aborts live work and remains a budget exhaustion', async () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + const started = deferred() + const root: Agent = { + name: 'deadline-root', + async act(task, scope: Scope): Promise { + const spawned = scope.spawn(blockingLeaf('blocked'), task, { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'blocked', + }) + expect(spawned.ok).toBe(true) + started.resolve() + const settled = await scope.next() + if (settled?.kind === 'down') throw new Error(settled.reason) + return settled?.out + }, + } + + const running = createSupervisor().run( + root, + 'task', + supervisorOpts({ budget: { maxIterations: 1, maxTokens: 10, deadlineMs: 25 } }), + ) + await started.promise + await vi.advanceTimersByTimeAsync(25) + + const result = await running + expect(result).toMatchObject({ kind: 'no-winner', reason: 'budget-exhausted' }) + expect(vi.getTimerCount()).toBe(0) + }) + + it('unrefs and clears the root timer when work finishes before the deadline', async () => { + const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout') + const clearTimeoutSpy = vi.spyOn(globalThis, 'clearTimeout') + const root: Agent = { + name: 'quick-root', + act: async () => 'done', + } + + const result = await createSupervisor().run( + root, + 'task', + supervisorOpts({ budget: { maxIterations: 1, maxTokens: 10, deadlineMs: 60_000 } }), + ) + + expect(result.kind).toBe('winner') + expect(setTimeoutSpy).toHaveBeenCalledTimes(1) + const timer = setTimeoutSpy.mock.results[0]?.value + expect(timer).toBeDefined() + expect(timer.hasRef()).toBe(false) + expect(clearTimeoutSpy).toHaveBeenCalledWith(timer) + }) + + it.each([ + { + name: 'the child duration is shorter', + parentDeadlineMs: 100, + childDeadlineMs: 20, + expectedMs: 20, + }, + { + name: 'the parent duration is shorter', + parentDeadlineMs: 20, + childDeadlineMs: 100, + expectedMs: 20, + }, + ])('aborts a child at the earlier cutoff when $name', async (testCase) => { + vi.useFakeTimers() + vi.setSystemTime(2_000) + const { scope } = await beginScope({ + maxIterations: 1, + maxTokens: 10, + deadlineMs: testCase.parentDeadlineMs, + }) + const spawned = scope.spawn(blockingLeaf('child'), 'task', { + budget: { + maxIterations: 1, + maxTokens: 10, + deadlineMs: testCase.childDeadlineMs, + }, + label: 'child', + }) + expect(spawned.ok).toBe(true) + + await vi.advanceTimersByTimeAsync(testCase.expectedMs - 1) + expect(scope.view.inFlight).toBe(1) + await vi.advanceTimersByTimeAsync(1) + + const settled = await scope.next() + expect(settled).toMatchObject({ kind: 'down', reason: 'aborted before settle' }) + expect(scope.view.inFlight).toBe(0) + expect(vi.getTimerCount()).toBe(0) + }) + + it('clears a child deadline when the child finishes first', async () => { + vi.useFakeTimers() + vi.setSystemTime(3_000) + const { scope } = await beginScope({ maxIterations: 1, maxTokens: 10 }) + const spawned = scope.spawn(immediateLeaf('child'), 'task', { + budget: { maxIterations: 1, maxTokens: 10, deadlineMs: 100 }, + label: 'child', + }) + expect(spawned.ok).toBe(true) + + expect((await scope.next())?.kind).toBe('done') + expect(vi.getTimerCount()).toBe(0) + }) +}) + +async function beginScope(budget: Budget): Promise<{ scope: Scope }> { + const journal = new InMemorySpawnJournal() + await journal.beginTree('deadline-scope', new Date(Date.now()).toISOString()) + return { + scope: createScope({ + parentId: 'deadline-scope', + root: 'deadline-scope', + pool: createBudgetPool(budget, Date.now), + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + seams: {}, + depth: 0, + signal: new AbortController().signal, + now: Date.now, + }), + } +} + +function blockingLeaf(name: string): Agent { + return leaf( + name, + (signal) => + new Promise>((resolve) => { + const finish = () => resolve({ outRef: `blocked:${name}`, out: name, spent: zeroSpend }) + if (signal.aborted) finish() + else signal.addEventListener('abort', finish, { once: true }) + }), + ) +} + +function immediateLeaf(name: string): Agent { + return leaf(name, async () => ({ outRef: `done:${name}`, out: name, spent: zeroSpend })) +} + +function teardownOnlyExecutor(teardown: Executor['teardown']): Executor { + return { + runtime: 'router', + execute: async () => ({ outRef: 'unused', out: undefined, spent: zeroSpend }), + teardown, + resultArtifact: () => ({ outRef: 'unused', out: undefined, spent: zeroSpend }), + } +} + +function leaf( + name: string, + execute: (signal: AbortSignal) => Promise>, +): Agent { + const executor: Executor = { + runtime: 'router', + execute: (_task, signal) => execute(signal), + teardown: async () => ({ destroyed: true }), + } + const executorSpec: AgentSpec = { + profile: { name } as AgentProfile, + harness: null, + executor, + } + return { name, act: async () => name, executorSpec } as Agent & { + executorSpec: AgentSpec + } +} + +function supervisorOpts(over: Partial = {}): SupervisorOpts { + return { + budget: over.budget ?? { maxIterations: 1, maxTokens: 10 }, + runId: 'deadline-supervisor', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + now: Date.now, + } +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} diff --git a/tests/kernel/supervise-full-profile-bridge.test.ts b/tests/kernel/supervise-full-profile-bridge.test.ts new file mode 100644 index 00000000..ce549294 --- /dev/null +++ b/tests/kernel/supervise-full-profile-bridge.test.ts @@ -0,0 +1,808 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { createServer, type Server, type ServerResponse } from 'node:http' +import type { AddressInfo } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it } from 'vitest' +import { InMemorySpawnJournal } from '../../src/durable/spawn-journal' +import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' +import { supervise } from '../../src/runtime/supervise/supervise' + +type BridgeRequest = { + model: string + run_id: string + session_id: string + agent_profile: AgentProfile + messages: Array<{ role: string; content: string }> +} + +const TEST_RUN_DIGEST = `sha256:${'c'.repeat(64)}` + +function numberSseDataFrames(body: string): string { + let seq = 0 + return body.replace(/^data: (?!\[DONE\])/gmu, () => `id: ${++seq}\ndata: `) +} + +function respondWithBridgeStream( + res: ServerResponse, + request: BridgeRequest, + stream: string, +): void { + res.writeHead(200, { + 'content-type': 'text/event-stream', + 'x-run-id': request.run_id, + 'x-run-request-digest': TEST_RUN_DIGEST, + }) + res.end(numberSseDataFrames(stream)) +} + +function cancelledRunId(url: string | undefined): string | undefined { + const match = url?.match(/^\/v1\/runs\/([^/]+)\/cancel(?:\?|$)/u) + return match?.[1] ? decodeURIComponent(match[1]) : undefined +} + +function respondWithTerminalCancellation(res: ServerResponse, runId: string): void { + res.writeHead(200, { + 'content-type': 'application/json', + 'x-run-id': runId, + 'x-run-request-digest': TEST_RUN_DIGEST, + }) + res.end( + JSON.stringify({ + terminal: true, + run: { + id: runId, + requestDigest: TEST_RUN_DIGEST, + terminal: true, + }, + }), + ) +} + +async function readJson(req: AsyncIterable): Promise { + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + return JSON.parse(Buffer.concat(chunks).toString('utf8')) as BridgeRequest +} + +async function callCoordination(url: string, name: string, args: unknown): Promise { + const response = await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: `${name}-${Date.now()}`, + method: 'tools/call', + params: { name, arguments: args }, + }), + }) + if (!response.ok) throw new Error(`coordination ${name} returned ${response.status}`) +} + +function successStream(content: string): string { + return [ + `data: ${JSON.stringify({ choices: [{ delta: { content } }] })}`, + `data: ${JSON.stringify({ usage: { prompt_tokens: 11, completion_tokens: 7, cost: 0.01 } })}`, + 'data: [DONE]', + '', + ].join('\n\n') +} + +function unknownCostStream(content: string): string { + return [ + `data: ${JSON.stringify({ choices: [{ delta: { content } }] })}`, + `data: ${JSON.stringify({ usage: { prompt_tokens: 11, completion_tokens: 7 } })}`, + 'data: [DONE]', + '', + ].join('\n\n') +} + +function unknownTokenStream(content: string): string { + return [ + `data: ${JSON.stringify({ choices: [{ delta: { content } }] })}`, + `data: ${JSON.stringify({ usage: { cost: 0.01 } })}`, + 'data: [DONE]', + '', + ].join('\n\n') +} + +describe('supervise — complete profiles over recursive cli-bridge managers', () => { + let server: Server | undefined + + afterEach(async () => { + if (server) await new Promise((resolve) => server?.close(resolve)) + server = undefined + }) + + it('runs PI → nested supervisor → worker without dropping any authored profile axis', async () => { + const requests: BridgeRequest[] = [] + const journal = new InMemorySpawnJournal() + const authorizations: Array<{ + depth: number + frozen: boolean + profile: AgentProfile + task: unknown + }> = [] + server = createServer(async (req, res) => { + try { + const body = await readJson(req) + requests.push(body) + const profile = body.agent_profile + const coordination = profile.mcp?.['agent-runtime-coordination'] + const depth = profile.metadata?.depth + + if (coordination?.enabled !== false && coordination?.url) { + if (depth === 0) { + await callCoordination(coordination.url, 'spawn_agent', { + profile: { + name: 'methods-supervisor', + description: 'Run the discriminating experiment', + harness: 'codex', + prompt: { systemPrompt: 'Supervise one empirical worker.' }, + model: { default: 'gpt-5.6', reasoningEffort: 'high' }, + tools: { shell: true }, + resources: { + skills: [ + { + kind: 'inline', + name: 'experimental-method', + content: '# Experimental method\nChange one variable at a time.', + }, + ], + failOnError: true, + }, + subagents: { + critic: { description: 'Find a confound', prompt: 'Challenge the result.' }, + }, + modes: { adversarial: { prompt: 'Try to falsify the claim.' } }, + metadata: { role: 'driver', depth: 1, family: 'scientific-method' }, + }, + task: 'Run one experiment and return its measured result.', + }) + } else { + await callCoordination(coordination.url, 'spawn_agent', { + profile: { + name: 'experiment-worker', + description: 'Execute and report the measurement', + harness: 'codex', + prompt: { systemPrompt: 'Return the exact measured result.' }, + model: { default: 'gpt-5.6', reasoningEffort: 'medium' }, + permissions: { shell: 'allow' }, + tools: { shell: true, web: false }, + resources: { + files: [ + { + path: 'protocol.txt', + resource: { + kind: 'inline', + name: 'protocol', + content: 'Measure twice; report both observations.', + }, + }, + ], + failOnError: true, + }, + metadata: { role: 'worker', depth: 2, family: 'scientific-method' }, + }, + task: 'Measure the system and report RESULT=42.', + }) + } + await callCoordination(coordination.url, 'await_event', {}) + } + + respondWithBridgeStream( + res, + body, + successStream(profile.name === 'experiment-worker' ? 'RESULT=42' : 'managed'), + ) + } catch (error) { + res.writeHead(500, { 'content-type': 'text/plain' }) + res.end(error instanceof Error ? error.message : String(error)) + } + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const backend: ExecutorConfig = { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + model: 'codex/gpt-5.6', + } + const rootProfile: AgentProfile = { + name: 'pi-leader', + description: 'Lead the full pursuit', + harness: 'codex', + prompt: { systemPrompt: 'Choose and supervise the most informative experiment.' }, + model: { default: 'gpt-5.6', reasoningEffort: 'xhigh' }, + tools: { web: true }, + mcp: { + literature: { transport: 'http', url: 'https://papers.example.test/mcp' }, + }, + resources: { + instructions: 'Keep hypotheses separate from observations.', + failOnError: true, + }, + metadata: { role: 'driver', depth: 0, family: 'discovery-native' }, + } + const rootTask = 'Resolve the pursuit with one measured experiment.' + + const result = await supervise(rootProfile, rootTask, { + backend, + budget: { maxIterations: 20, maxTokens: 100_000 }, + perWorker: { maxIterations: 8, maxTokens: 20_000 }, + maxDepth: 4, + deliverable: { + check: (out) => + typeof out === 'object' && + out !== null && + (out as { content?: unknown }).content === 'RESULT=42', + describe: 'worker reports RESULT=42', + }, + profileSecurity: { + allowLocalMcp: false, + allowHooks: false, + allowedMcpHosts: ['papers.example.test'], + }, + journal, + runId: 'identity-run', + execution: { + candidateDigest: canonicalCandidateDigest({ candidate: 'pi-leader' }), + correlation: { pursuitId: 'pursuit-1', experimentId: 'experiment-root' }, + }, + authorizeSpawn: (input) => { + authorizations.push({ + depth: input.depth, + frozen: + Object.isFrozen(input) && + Object.isFrozen(input.profile) && + Object.isFrozen(input.task) && + Object.isFrozen(input.budget), + profile: input.profile, + task: input.task, + }) + const name = input.profile.name ?? 'unnamed' + return { + profile: input.profile, + execution: { + candidateDigest: canonicalCandidateDigest({ candidate: name }), + correlation: { + pursuitId: 'pursuit-1', + experimentId: `experiment-${input.depth}`, + }, + }, + } + }, + }) + + expect(result.kind).toBe('winner') + expect(requests.map((request) => request.agent_profile.name)).toEqual([ + 'pi-leader', + 'methods-supervisor', + 'experiment-worker', + ]) + expect(requests.map((request) => request.model)).toEqual([ + 'codex/gpt-5.6', + 'codex/gpt-5.6', + 'codex/gpt-5.6', + ]) + expect(requests.map((request) => request.session_id)).toEqual([ + expect.stringMatching(/^supervised-manager-[a-f0-9]{64}$/), + expect.stringMatching(/^supervised-manager-[a-f0-9]{64}$/), + expect.stringMatching(/^supervised-worker-[a-f0-9]{64}$/), + ]) + expect(new Set(requests.map((request) => request.session_id)).size).toBe(3) + + const pi = requests[0]!.agent_profile + expect(pi.tools).toEqual({ web: true }) + expect(pi.mcp?.literature).toEqual({ + transport: 'http', + url: 'https://papers.example.test/mcp', + }) + expect(pi.mcp?.['agent-runtime-coordination']).toMatchObject({ transport: 'http' }) + + const nested = requests[1]!.agent_profile + expect(nested.resources?.skills?.[0]).toMatchObject({ name: 'experimental-method' }) + expect(nested.subagents?.critic?.prompt).toBe('Challenge the result.') + expect(nested.modes?.adversarial?.prompt).toBe('Try to falsify the claim.') + expect(nested.mcp?.['agent-runtime-coordination']).toMatchObject({ transport: 'http' }) + + const worker = requests[2]!.agent_profile + expect(worker.permissions).toEqual({ shell: 'allow' }) + expect(worker.resources?.files?.[0]?.path).toBe('protocol.txt') + expect(worker.mcp?.['agent-runtime-coordination']).toBeUndefined() + expect(requests.every((request) => request.messages[0]?.role === 'user')).toBe(true) + expect(result.spentTotal.tokens).toEqual({ input: 33, output: 21 }) + expect(result.spentTotal.iterations).toBe(1) + expect(authorizations).toEqual([ + { + depth: 1, + frozen: true, + profile: expect.objectContaining({ name: 'methods-supervisor' }), + task: 'Run one experiment and return its measured result.', + }, + { + depth: 2, + frozen: true, + profile: expect.objectContaining({ name: 'experiment-worker' }), + task: 'Measure the system and report RESULT=42.', + }, + ]) + + const rootEvents = await journal.loadTree('identity-run') + expect(JSON.stringify(rootEvents)).not.toContain(backend.bridgeUrl) + expect(JSON.stringify(rootEvents)).not.toContain(backend.bridgeBearer) + expect(JSON.stringify(rootEvents)).not.toContain( + String(pi.mcp?.['agent-runtime-coordination']?.url), + ) + const rootSpawn = rootEvents?.find( + (event) => event.kind === 'spawned' && event.id === 'identity-run', + ) + const nestedSpawn = rootEvents?.find( + (event) => event.kind === 'spawned' && event.id !== 'identity-run', + ) + expect(rootSpawn?.identity).toEqual({ + profileDigest: canonicalCandidateDigest(rootProfile), + taskDigest: canonicalCandidateDigest(rootTask), + candidateDigest: canonicalCandidateDigest({ candidate: 'pi-leader' }), + correlation: { pursuitId: 'pursuit-1', experimentId: 'experiment-root' }, + }) + const rootMaterialized = rootEvents?.find((event) => event.kind === 'materialized') + expect(rootMaterialized).toMatchObject({ + kind: 'materialized', + id: 'identity-run', + receipt: { + status: 'known', + authoredProfileDigest: canonicalCandidateDigest(rootProfile), + effectiveProfileDigest: canonicalCandidateDigest(rootProfile), + runtime: 'cli', + backend: 'bridge', + model: { status: 'known', id: 'codex/gpt-5.6' }, + execution: { kind: 'session', id: requests[0]!.session_id }, + }, + }) + expect( + rootMaterialized?.kind === 'materialized' + ? rootMaterialized.receipt.platformAttachmentsDigest + : undefined, + ).toMatch(/^sha256:[a-f0-9]{64}$/) + expect( + rootEvents?.find((event) => event.kind === 'execution-bound' && event.id === 'identity-run'), + ).toMatchObject({ + binding: { + status: 'known', + descriptor: { kind: 'bridge-session', transport: 'http', coordination: true }, + }, + }) + expect(nestedSpawn?.identity).toEqual({ + profileDigest: canonicalCandidateDigest(authorizations[0]!.profile), + taskDigest: canonicalCandidateDigest(authorizations[0]!.task), + candidateDigest: canonicalCandidateDigest({ candidate: 'methods-supervisor' }), + correlation: { pursuitId: 'pursuit-1', experimentId: 'experiment-1' }, + }) + const nestedEvents = await journal.loadTree('identity-run/identity-run:s0') + expect(JSON.stringify(nestedEvents)).not.toContain(backend.bridgeUrl) + expect(JSON.stringify(nestedEvents)).not.toContain(backend.bridgeBearer) + const workerSpawn = nestedEvents?.find((event) => event.kind === 'spawned') + expect(workerSpawn?.identity).toEqual({ + profileDigest: canonicalCandidateDigest(authorizations[1]!.profile), + taskDigest: canonicalCandidateDigest(authorizations[1]!.task), + candidateDigest: canonicalCandidateDigest({ candidate: 'experiment-worker' }), + correlation: { pursuitId: 'pursuit-1', experimentId: 'experiment-2' }, + }) + }) + + it('isolates identical concurrent managers but reuses one durable manager session on restart', async () => { + const sessions: string[] = [] + server = createServer(async (req, res) => { + const body = await readJson(req) + sessions.push(body.session_id) + respondWithBridgeStream(res, body, successStream('managed')) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const profile = { + name: 'pi-leader', + harness: 'codex', + prompt: { systemPrompt: 'Lead the pursuit.' }, + model: { default: 'gpt-5.6' }, + } as const + const backend = { + backend: 'bridge' as const, + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + model: 'codex/gpt-5.6', + } + const task = 'Choose the next experiment.' + + await Promise.all([ + supervise(profile, task, { + backend, + budget: { maxIterations: 4, maxTokens: 10_000 }, + runId: 'same-visible-run-id', + }), + supervise(profile, task, { + backend, + budget: { maxIterations: 4, maxTokens: 10_000 }, + runId: 'same-visible-run-id', + }), + ]) + expect(sessions).toHaveLength(2) + expect(new Set(sessions).size).toBe(2) + + sessions.length = 0 + const runDir = await mkdtemp(join(tmpdir(), 'manager-stable-session-')) + try { + const options = { + backend, + budget: { maxIterations: 4, maxTokens: 10_000 }, + runDir, + runId: 'durable-manager-session', + } + await supervise(profile, task, options) + await supervise(profile, task, options) + + expect(sessions).toHaveLength(2) + expect(sessions[0]).toMatch(/^supervised-manager-[a-f0-9]{64}$/) + expect(sessions[1]).toBe(sessions[0]) + } finally { + await rm(runDir, { recursive: true, force: true }) + } + }) + + it('captures mutable supervision policy, profiles, limits, and callback selection at intake', async () => { + const requests: BridgeRequest[] = [] + server = createServer(async (req, res) => { + const body = await readJson(req) + requests.push(body) + respondWithBridgeStream(res, body, successStream('RESULT=42')) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + + let releaseFirstTurn!: () => void + const firstTurnHeld = new Promise((resolve) => { + releaseFirstTurn = resolve + }) + let markFirstTurnEntered!: () => void + const firstTurnEntered = new Promise((resolve) => { + markFirstTurnEntered = resolve + }) + let turn = 0 + const callbackCalls: string[] = [] + const brain = async () => { + turn += 1 + if (turn === 1) { + markFirstTurnEntered() + await firstTurnHeld + return { + toolCalls: [ + { + id: 'spawn', + name: 'spawn_agent', + arguments: JSON.stringify({ + profile: { + name: 'policy-worker', + harness: 'codex', + model: { default: 'safe-model' }, + mcp: { + allowed: { transport: 'http', url: 'https://allowed.test/mcp' }, + }, + }, + task: 'Return RESULT=42.', + }), + }, + ], + usage: { input: 1, output: 1 }, + costUsd: 0, + } + } + if (turn === 2) { + return { + toolCalls: [{ id: 'await', name: 'await_event', arguments: JSON.stringify({}) }], + usage: { input: 1, output: 1 }, + costUsd: 0, + } + } + return { content: 'done', toolCalls: [], usage: { input: 1, output: 1 }, costUsd: 0 } + } + const rootProfile: AgentProfile = { + name: 'original-root', + harness: 'cli-base', + prompt: { systemPrompt: 'Use the worker.' }, + } + const backend = { + backend: 'bridge' as const, + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + model: 'safe-model', + } + const profileSecurity = { + allowLocalMcp: false, + allowHooks: false, + allowedMcpHosts: ['allowed.test'], + } + const perWorker = { maxIterations: 2, maxTokens: 100 } + const allowedModels = ['safe-model'] + const deliverable = { + check: (out: unknown) => + typeof out === 'object' && + out !== null && + (out as { content?: unknown }).content === 'RESULT=42', + describe: 'the measured result', + } + const options = { + backend, + budget: { maxIterations: 10, maxTokens: 10_000 }, + perWorker, + profileSecurity, + allowedModels, + deliverable, + brain, + maxTurns: 4, + authorizeSpawn: (input: { + profile: AgentProfile + parent: AgentProfile + task: unknown + budget: { maxIterations: number; maxTokens: number } + }) => { + callbackCalls.push('original-authorizer') + expect(input.parent.name).toBe('original-root') + expect(input.budget).toMatchObject({ maxIterations: 2, maxTokens: 100 }) + return { profile: input.profile } + }, + isDriverProfile: () => false, + } + + const run = supervise(rootProfile, { pursuit: 'original-task' }, options) + await firstTurnEntered + rootProfile.name = 'mutated-root' + backend.bridgeUrl = 'http://127.0.0.1:1' + backend.model = 'mutated-model' + profileSecurity.allowedMcpHosts.splice(0, 1, 'mutated.test') + perWorker.maxIterations = 0 + perWorker.maxTokens = 1 + allowedModels.splice(0, 1, 'mutated-model') + deliverable.check = () => false + options.maxTurns = 1 + options.authorizeSpawn = (input) => { + callbackCalls.push('replacement-authorizer') + return { profile: input.profile } + } + options.isDriverProfile = () => true + options.brain = async () => ({ content: 'replacement', toolCalls: [] }) + releaseFirstTurn() + + const result = await run + expect(result.kind).toBe('winner') + expect(callbackCalls).toEqual(['original-authorizer']) + expect(requests).toHaveLength(1) + expect(requests[0]).toMatchObject({ + model: 'codex/safe-model', + agent_profile: { + name: 'policy-worker', + mcp: { allowed: { url: 'https://allowed.test/mcp' } }, + }, + }) + expect(requests[0]?.agent_profile.mcp?.['agent-runtime-coordination']).toBeUndefined() + }) + + it('refuses a backend overlay before it can occupy the coordination alias', () => { + expect(() => + supervise( + { + name: 'pi-leader', + harness: 'codex', + prompt: { systemPrompt: 'Lead the pursuit.' }, + model: { default: 'gpt-5.6' }, + }, + 'Choose the next experiment.', + { + backend: { + backend: 'bridge', + bridgeUrl: 'http://127.0.0.1:1', + bridgeBearer: 'unused', + model: 'codex/gpt-5.6', + agentProfile: { + mcp: { + 'agent-runtime-coordination': { + transport: 'http', + url: 'http://169.254.169.254/latest/meta-data', + }, + }, + }, + }, + budget: { maxIterations: 2, maxTokens: 10_000 }, + }, + ), + ).toThrow(/backend agentProfile overlays are not allowed/) + }) + + it('refuses a manager with unknown cost under a dollar-capped budget', async () => { + server = createServer(async (req, res) => { + const body = await readJson(req) + respondWithBridgeStream(res, body, unknownCostStream('managed')) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + + const result = await supervise( + { + name: 'pi-leader', + harness: 'codex', + prompt: { systemPrompt: 'Lead the pursuit.' }, + model: { default: 'gpt-5.6' }, + }, + 'Choose the next experiment.', + { + backend: { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + model: 'codex/gpt-5.6', + }, + budget: { maxIterations: 2, maxTokens: 10_000, maxUsd: 1 }, + }, + ) + + expect(result.kind).toBe('no-winner') + if (result.kind !== 'no-winner') return + expect(result.reason).toBe('budget-exhausted') + expect(result.spentTotal).toMatchObject({ usd: 0, usdKnown: false }) + }) + + it('refuses a manager with unknown token usage under the always-capped token budget', async () => { + server = createServer(async (req, res) => { + const body = await readJson(req) + respondWithBridgeStream(res, body, unknownTokenStream('managed')) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + + const result = await supervise( + { + name: 'pi-leader', + harness: 'codex', + prompt: { systemPrompt: 'Lead the pursuit.' }, + model: { default: 'gpt-5.6' }, + }, + 'Choose the next experiment.', + { + backend: { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + model: 'codex/gpt-5.6', + }, + budget: { maxIterations: 2, maxTokens: 10_000 }, + }, + ) + + expect(result.kind).toBe('no-winner') + if (result.kind !== 'no-winner') return + expect(result.reason).toBe('budget-exhausted') + expect(result.spentTotal).toMatchObject({ + tokens: { input: 0, output: 0 }, + tokensKnown: false, + }) + }) + + it('records a partial manager stream as unknown and refuses a replacement after restart', async () => { + const runDir = await mkdtemp(join(tmpdir(), 'manager-partial-stream-')) + let requests = 0 + try { + server = createServer(async (req, res) => { + const cancelled = cancelledRunId(req.url) + if (cancelled !== undefined) { + respondWithTerminalCancellation(res, cancelled) + return + } + const body = await readJson(req) + requests += 1 + res.writeHead(200, { + 'content-type': 'text/event-stream', + 'x-run-id': body.run_id, + 'x-run-request-digest': TEST_RUN_DIGEST, + }) + res.write( + `id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 13, completion_tokens: 5, cost: 0.02 } })}\n\n`, + ) + setTimeout(() => res.socket?.destroy(new Error('socket died before terminal receipt')), 10) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const profile = { + name: 'pi-leader', + harness: 'codex', + prompt: { systemPrompt: 'Lead the pursuit.' }, + model: { default: 'gpt-5.6' }, + } as const + const options = { + backend: { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + model: 'codex/gpt-5.6', + } as const, + budget: { maxIterations: 2, maxTokens: 100, maxUsd: 1 }, + runDir, + runId: 'partial-manager-restart', + } + + const first = await supervise(profile, 'Choose the next experiment.', options) + expect(first.kind).toBe('no-winner') + expect(first.spentTotal).toMatchObject({ + tokens: { input: 13, output: 5 }, + tokensKnown: false, + usd: 0.02, + usdKnown: false, + }) + + const resumed = await supervise(profile, 'Choose the next experiment.', options) + expect(resumed.kind).toBe('no-winner') + expect(resumed.spentTotal).toMatchObject({ + tokens: { input: 13, output: 5 }, + tokensKnown: false, + usd: 0.02, + usdKnown: false, + }) + // One reconnect under the same durable run id proves the first transport loss before the + // repeated event id fails continuity. Resume itself starts no replacement manager. + expect(requests).toBe(2) + } finally { + await rm(runDir, { recursive: true, force: true }) + } + }) + + it("preserves a leaf's unknown dollar cost and refuses it under a dollar cap", async () => { + const requests: BridgeRequest[] = [] + server = createServer(async (req, res) => { + const body = await readJson(req) + requests.push(body) + const profile = body.agent_profile + const coordination = profile.mcp?.['agent-runtime-coordination'] + if (coordination?.enabled !== false && coordination?.url) { + await callCoordination(coordination.url, 'spawn_agent', { + profile: { + name: 'worker', + harness: 'codex', + prompt: { systemPrompt: 'Return the result.' }, + model: { default: 'gpt-5.6' }, + }, + task: 'Return RESULT=42.', + }) + await callCoordination(coordination.url, 'await_event', {}) + } + respondWithBridgeStream( + res, + body, + profile.name === 'worker' ? unknownCostStream('RESULT=42') : successStream('managed'), + ) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + + const result = await supervise( + { + name: 'pi-leader', + harness: 'codex', + prompt: { systemPrompt: 'Lead the pursuit.' }, + model: { default: 'gpt-5.6' }, + }, + 'Choose the next experiment.', + { + backend: { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + model: 'codex/gpt-5.6', + }, + budget: { maxIterations: 4, maxTokens: 10_000, maxUsd: 1 }, + }, + ) + + expect(requests.map((request) => request.agent_profile.name)).toEqual(['pi-leader', 'worker']) + expect(result.kind).toBe('no-winner') + expect(result.spentTotal.usdKnown).toBe(false) + }) +}) diff --git a/tests/kernel/supervise-restart-resource-safety.test.ts b/tests/kernel/supervise-restart-resource-safety.test.ts new file mode 100644 index 00000000..1ba883cb --- /dev/null +++ b/tests/kernel/supervise-restart-resource-safety.test.ts @@ -0,0 +1,1295 @@ +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' +import { driverChild, withDriverExecutor } from '../../src/runtime/supervise/driver-executor' +import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { createSupervisor } from '../../src/runtime/supervise/supervisor' +import type { + Agent, + AgentSpec, + Executor, + ExecutorResult, + NodeExecutionIdentity, + Scope, + SpawnEvent, + SpawnJournal, + Spend, + UsageEvent, +} from '../../src/runtime/supervise/types' + +const zeroSpend: Spend = { + iterations: 0, + tokens: { input: 0, output: 0 }, + usd: 0, + ms: 0, +} + +describe('supervision restart and resource safety', () => { + it('snapshots the root task, budget, and identity before delayed journal intake', async () => { + const base = new InMemorySpawnJournal() + const beginEntered = deferred() + const releaseBegin = deferred() + const journal: SpawnJournal = { + loadTree: (root) => base.loadTree(root), + async beginTree(root, at) { + beginEntered.resolve() + await releaseBegin.promise + return base.beginTree(root, at) + }, + appendEvent: (root, event) => base.appendEvent(root, event), + } + const task = { instruction: 'AUTHORIZED' } + const budget = { maxIterations: 1, maxTokens: 10 } + const expectedIdentity = { + profileDigest: canonicalCandidateDigest({ name: 'root-profile' }), + taskDigest: canonicalCandidateDigest({ instruction: 'AUTHORIZED' }), + } + const rootIdentity = { ...expectedIdentity } + let rootObservation: + | { instruction: string; frozen: boolean; tokensLeft: number; tokensKnown: boolean } + | undefined + const running = createSupervisor().run( + { + name: 'root', + act: async (authorizedTask, scope) => { + rootObservation = { + instruction: authorizedTask.instruction, + frozen: Object.isFrozen(authorizedTask), + tokensLeft: scope.budget.tokensLeft, + tokensKnown: scope.budget.tokensKnown, + } + return authorizedTask.instruction + }, + }, + task, + { + budget, + rootIdentity, + runId: 'immutable-root-input', + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }, + ) + + await beginEntered.promise + task.instruction = 'MUTATED' + budget.maxTokens = 100 + rootIdentity.taskDigest = canonicalCandidateDigest({ instruction: 'MUTATED' }) + releaseBegin.resolve() + + const result = await running + const events = (await base.loadTree('immutable-root-input')) ?? [] + const rootEvent = events.find((event) => event.kind === 'spawned' && event.parent === undefined) + + expect(result.kind).toBe('winner') + if (result.kind !== 'winner') return + expect(result.out).toBe('AUTHORIZED') + expect(rootObservation).toEqual({ + instruction: 'AUTHORIZED', + frozen: true, + tokensLeft: 10, + tokensKnown: true, + }) + expect(rootEvent).toMatchObject({ + budget: { maxIterations: 1, maxTokens: 10 }, + identity: expectedIdentity, + }) + expect(Object.isFrozen(rootEvent?.budget)).toBe(true) + expect(rootEvent?.identity?.taskDigest).toBe( + canonicalCandidateDigest({ instruction: 'AUTHORIZED' }), + ) + }) + + it('refuses malformed fresh root identities before journaling or acting', async () => { + const validProfile = canonicalCandidateDigest({ name: 'root-profile' }) + const validTask = canonicalCandidateDigest({ instruction: 'AUTHORIZED' }) + const cases: ReadonlyArray = [ + ['profile digest', { profileDigest: 'sha256:invalid', taskDigest: validTask }], + ['task digest', { profileDigest: validProfile, taskDigest: 'sha256:invalid' }], + [ + 'candidate digest', + { + profileDigest: validProfile, + taskDigest: validTask, + candidateDigest: 'sha256:invalid', + }, + ], + [ + 'correlation', + { + profileDigest: validProfile, + taskDigest: validTask, + correlation: { worker: '' }, + }, + ], + [ + 'unknown field', + { + profileDigest: validProfile, + taskDigest: validTask, + injected: true, + } as NodeExecutionIdentity, + ], + ] + + for (const [label, rootIdentity] of cases) { + const base = new InMemorySpawnJournal() + let loads = 0 + let begins = 0 + let appends = 0 + let acts = 0 + const journal: SpawnJournal = { + async loadTree(root) { + loads += 1 + return base.loadTree(root) + }, + async beginTree(root, at) { + begins += 1 + return base.beginTree(root, at) + }, + async appendEvent(root, event) { + appends += 1 + return base.appendEvent(root, event) + }, + } + + await expect( + createSupervisor().run( + { + name: 'root', + act: async () => { + acts += 1 + return 'unsafe' + }, + }, + 'task', + { + budget: { maxIterations: 1, maxTokens: 10 }, + rootIdentity, + runId: `malformed-root-${label}`, + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }, + ), + label, + ).rejects.toThrow(/requires an exact rootIdentity/) + expect({ acts, loads, begins, appends }, label).toEqual({ + acts: 0, + loads: 0, + begins: 0, + appends: 0, + }) + } + }) + + it('classifies exhausted iterations without rejecting a valid zero-work winner', async () => { + let factoryCalls = 0 + let spawnReason: string | undefined + const exhausted = await createSupervisor().run( + { + name: 'iteration-exhausted-root', + act: async (task, scope) => { + const spawned = scope.spawn( + () => { + factoryCalls += 1 + return resultLeaf('must-not-run', zeroSpend) + }, + task, + { + budget: { maxIterations: 1, maxTokens: 1 }, + label: 'must-not-run', + }, + ) + if (!spawned.ok) spawnReason = spawned.reason + return undefined + }, + }, + 'task', + { + budget: { maxIterations: 0, maxTokens: 10 }, + runId: 'iteration-exhausted-root', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }, + ) + + expect(exhausted).toMatchObject({ kind: 'no-winner', reason: 'budget-exhausted' }) + expect(spawnReason).toBe('budget-exhausted') + expect(factoryCalls).toBe(0) + + const valid = await createSupervisor().run( + { name: 'zero-work-root', act: async () => 'valid zero-work result' }, + 'task', + { + budget: { maxIterations: 0, maxTokens: 10 }, + runId: 'valid-zero-work-root', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }, + ) + expect(valid).toMatchObject({ kind: 'winner', out: 'valid zero-work result' }) + }) + + it('executes the immutable task snapshot whose digest was authorized', async () => { + let executorSaw: unknown + let taskWasFrozen = false + const leaf = leafFromExecutor('task-snapshot', () => ({ + runtime: 'router', + async execute(task): Promise> { + executorSaw = (task as { instruction: string }).instruction + taskWasFrozen = Object.isFrozen(task) + return { outRef: 'internal', out: String(executorSaw), spent: zeroSpend } + }, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => ({ outRef: 'internal', out: String(executorSaw), spent: zeroSpend }), + })) + const root: Agent = { + name: 'root', + async act(_task, scope) { + const childTask = { instruction: 'AUTHORIZED' } + const expectedDigest = canonicalCandidateDigest(childTask) + const spawned = scope.spawn(leaf, childTask, { + key: 'task-snapshot', + budget: { maxIterations: 1, maxTokens: 2 }, + label: 'task-snapshot', + }) + expect(spawned.ok).toBe(true) + childTask.instruction = 'MUTATED' + const settled = await scope.next() + return { + digest: spawned.ok ? spawned.handle.identity?.taskDigest : undefined, + out: settled?.kind === 'done' ? settled.out : undefined, + } + }, + } + + const result = await createSupervisor().run( + root, + 'task', + { + budget: { maxIterations: 1, maxTokens: 2 }, + runId: 'immutable-task-snapshot', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }, + ) + + expect(result.kind).toBe('winner') + if (result.kind !== 'winner') return + expect(result.out).toEqual({ + digest: canonicalCandidateDigest({ instruction: 'AUTHORIZED' }), + out: 'AUTHORIZED', + }) + expect(executorSaw).toBe('AUTHORIZED') + expect(taskWasFrozen).toBe(true) + }) + + it('does not let a caller mutate a driver allocation after it was reserved', async () => { + const journal = new InMemorySpawnJournal() + let leafExecutions = 0 + const costlyLeaf = leafFromExecutor('costly-leaf', () => ({ + runtime: 'router', + async execute(): Promise> { + leafExecutions += 1 + return { + outRef: 'internal', + out: 'costly', + spent: { ...zeroSpend, tokens: { input: 50, output: 0 } }, + } + }, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => ({ + outRef: 'internal', + out: 'costly', + spent: { ...zeroSpend, tokens: { input: 50, output: 0 } }, + }), + })) + const nested: Agent = { + name: 'nested', + async act(task, scope): Promise { + const child = scope.spawn(costlyLeaf, task, { + budget: { maxIterations: 1, maxTokens: 50 }, + label: 'costly-leaf', + }) + if (!child.ok) return child.reason + await scope.next() + return 'accepted' + }, + } + const root: Agent = { + name: 'root', + async act(task, scope): Promise { + const allocation = { maxIterations: 1, maxTokens: 10 } + const manager = scope.spawn( + driverChild( + { name: 'nested-manager', harness: 'cli-base', metadata: { role: 'driver' } }, + nested, + journal, + ), + task, + { budget: allocation, label: 'nested-manager' }, + ) + expect(manager.ok).toBe(true) + allocation.maxTokens = 100 + const settled = await scope.next() + return settled?.kind === 'done' ? String(settled.out) : 'manager-down' + }, + } + + const result = await createSupervisor().run(root, 'task', { + budget: { maxIterations: 1, maxTokens: 10 }, + maxDepth: 2, + runId: 'immutable-driver-budget', + journal, + blobs: new InMemoryResultBlobStore(), + executors: withDriverExecutor(createExecutorRegistry()), + }) + + expect(result.kind).toBe('winner') + if (result.kind !== 'winner') return + expect(result.out).toBe('budget-exhausted') + expect(leafExecutions).toBe(0) + expect(result.spentTotal.tokens).toEqual({ input: 0, output: 0 }) + }) + + it('refuses a resumed root whose profile/task/candidate identity changed', async () => { + const journal = new InMemorySpawnJournal() + const blobs = new InMemoryResultBlobStore() + const executors = createExecutorRegistry() + const budget = { maxIterations: 2, maxTokens: 100 } + const firstIdentity = { + profileDigest: `sha256:${'1'.repeat(64)}`, + taskDigest: `sha256:${'2'.repeat(64)}`, + } + const secondIdentity = { + profileDigest: `sha256:${'3'.repeat(64)}`, + taskDigest: `sha256:${'4'.repeat(64)}`, + } + await createSupervisor().run( + { name: 'root-a', act: async () => 'A' }, + 'task A', + { + budget, + rootIdentity: firstIdentity, + runId: 'resume-identity', + journal, + blobs, + executors, + resume: true, + }, + ) + let secondActs = 0 + await expect( + createSupervisor().run( + { + name: 'root-b', + act: async () => { + secondActs += 1 + return 'B' + }, + }, + 'task B', + { + budget, + rootIdentity: secondIdentity, + runId: 'resume-identity', + journal, + blobs, + executors, + resume: true, + }, + ), + ).rejects.toThrow(/resume identity mismatch/) + expect(secondActs).toBe(0) + }) + + it('refuses resume without an exact root identity before reading, mutating, or acting', async () => { + const base = new InMemorySpawnJournal() + let loads = 0 + let begins = 0 + let appends = 0 + const journal: SpawnJournal = { + async loadTree(root) { + loads += 1 + return base.loadTree(root) + }, + async beginTree(root, at) { + begins += 1 + return base.beginTree(root, at) + }, + async appendEvent(root, event) { + appends += 1 + return base.appendEvent(root, event) + }, + } + let acts = 0 + + await expect( + createSupervisor().run( + { + name: 'unsafe-root', + act: async () => { + acts += 1 + return 'unsafe' + }, + }, + 'unsafe task', + { + budget: { maxIterations: 1, maxTokens: 10 }, + runId: 'missing-root-identity', + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + resume: true, + }, + ), + ).rejects.toThrow(/requires an exact rootIdentity/) + + await expect( + createSupervisor().run( + { + name: 'partial-root', + act: async () => { + acts += 1 + return 'partial' + }, + }, + 'partial task', + { + budget: { maxIterations: 1, maxTokens: 10 }, + rootIdentity: { profileDigest: `sha256:${'4'.repeat(64)}` as const }, + runId: 'partial-root-identity', + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + resume: true, + }, + ), + ).rejects.toThrow(/requires an exact rootIdentity/) + + expect({ acts, loads, begins, appends }).toEqual({ acts: 0, loads: 0, begins: 0, appends: 0 }) + }) + + it('does not let an omitted identity bypass a changed root and task on the same run id', async () => { + const journal = new InMemorySpawnJournal() + const blobs = new InMemoryResultBlobStore() + const executors = createExecutorRegistry() + const runId = 'omitted-resume-identity' + const rootIdentity = { + profileDigest: `sha256:${'5'.repeat(64)}` as const, + taskDigest: `sha256:${'6'.repeat(64)}` as const, + } + let firstActs = 0 + await createSupervisor().run( + { + name: 'root-a', + act: async () => { + firstActs += 1 + return 'A' + }, + }, + 'task A', + { + budget: { maxIterations: 1, maxTokens: 10 }, + rootIdentity, + runId, + journal, + blobs, + executors, + resume: true, + }, + ) + const before = await journal.loadTree(runId) + let secondActs = 0 + + await expect( + createSupervisor().run( + { + name: 'root-b', + act: async () => { + secondActs += 1 + return 'B' + }, + }, + 'task B', + { + budget: { maxIterations: 1, maxTokens: 10 }, + runId, + journal, + blobs, + executors, + resume: true, + }, + ), + ).rejects.toThrow(/requires an exact rootIdentity/) + + expect(firstActs).toBe(1) + expect(secondActs).toBe(0) + expect(await journal.loadTree(runId)).toEqual(before) + }) + + it('does not execute a child until its identity event is committed', async () => { + const base = new InMemorySpawnJournal() + const releaseCommit = deferred() + const journal: SpawnJournal = { + loadTree: (root) => base.loadTree(root), + beginTree: (root, at) => base.beginTree(root, at), + async appendEvent(root, event): Promise { + if (event.kind === 'spawned' && event.parent !== undefined) { + await releaseCommit.promise + } + await base.appendEvent(root, event) + }, + } + let executions = 0 + let executionsBeforeCommit = -1 + const leaf = leafFromExecutor('commit-first', () => ({ + runtime: 'router', + async execute(): Promise> { + executions += 1 + return { outRef: 'internal', out: 'done', spent: zeroSpend } + }, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => ({ outRef: 'internal', out: 'done', spent: zeroSpend }), + })) + const result = await createSupervisor().run( + { + name: 'root', + async act(task, scope): Promise { + const spawned = scope.spawn(leaf, task, { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'commit-first', + }) + expect(spawned.ok).toBe(true) + await Promise.resolve() + executionsBeforeCommit = executions + releaseCommit.resolve() + await scope.next() + return 'root result' + }, + }, + 'task', + { + budget: { maxIterations: 1, maxTokens: 10 }, + runId: 'commit-before-execute', + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }, + ) + expect(result.kind).toBe('winner') + expect(executionsBeforeCommit).toBe(0) + expect(executions).toBe(1) + }) + + it('restores prior spend and the original absolute deadline when a root resumes', async () => { + const journal = new InMemorySpawnJournal() + const blobs = new InMemoryResultBlobStore() + const executors = createExecutorRegistry() + const runId = 'resume-root-limits' + const rootIdentity = { + profileDigest: `sha256:${'7'.repeat(64)}` as const, + taskDigest: `sha256:${'8'.repeat(64)}` as const, + } + let nowMs = 1_000 + + const firstRoot: Agent = { + name: 'first-root', + async act(task, scope): Promise { + const spawned = scope.spawn( + resultLeaf('prior-work', { + iterations: 1, + tokens: { input: 80, output: 0 }, + usd: 0, + ms: 1, + }), + task, + { + budget: { maxIterations: 1, maxTokens: 100 }, + label: 'prior-work', + }, + ) + expect(spawned.ok).toBe(true) + expect((await scope.next())?.kind).toBe('done') + return 'first result' + }, + } + + const first = await createSupervisor().run(firstRoot, 'task', { + budget: { maxIterations: 2, maxTokens: 100, deadlineMs: 100 }, + rootIdentity, + runId, + journal, + blobs, + executors, + now: () => nowMs, + }) + expect(first.kind).toBe('winner') + + nowMs = 1_090 + let factoryCalls = 0 + let observed: + | { + tokensLeft: number + tokensKnown: boolean + deadlineMs: number + attempt: ReturnType['spawn']> + } + | undefined + const resumedRoot: Agent = { + name: 'resumed-root', + act(task, scope): Promise { + const before = scope.budget + const attempt = scope.spawn( + () => { + factoryCalls += 1 + return resultLeaf('must-not-run', zeroSpend) + }, + task, + { + budget: { maxIterations: 1, maxTokens: 21 }, + label: 'must-not-run', + }, + ) + observed = { + tokensLeft: before.tokensLeft, + tokensKnown: before.tokensKnown, + deadlineMs: before.deadlineMs, + attempt, + } + return Promise.resolve('resumed result') + }, + } + + const resumed = await createSupervisor().run(resumedRoot, 'task', { + budget: { maxIterations: 2, maxTokens: 100, deadlineMs: 100 }, + rootIdentity, + runId, + journal, + blobs, + executors, + resume: true, + now: () => nowMs, + }) + + expect(resumed.kind).toBe('winner') + expect(observed).toMatchObject({ + tokensLeft: 20, + tokensKnown: true, + deadlineMs: 1_100, + attempt: { ok: false, reason: 'budget-exhausted' }, + }) + expect(factoryCalls).toBe(0) + expect(resumed.spentTotal).toMatchObject({ + iterations: 1, + tokens: { input: 80, output: 0 }, + }) + }) + + it('anchors a fresh deadline before delayed beginTree and preserves it exactly on resume', async () => { + const base = new InMemorySpawnJournal() + let nowMs = 4_000 + const journal: SpawnJournal = { + loadTree: (root) => base.loadTree(root), + async beginTree(root, at) { + nowMs += 75 + await Promise.resolve() + return base.beginTree(root, at) + }, + appendEvent: (root, event) => base.appendEvent(root, event), + } + const budget = { maxIterations: 1, maxTokens: 10, deadlineMs: 100 } + const rootIdentity = { + profileDigest: `sha256:${'9'.repeat(64)}` as const, + taskDigest: `sha256:${'a'.repeat(64)}` as const, + } + let firstDeadline = 0 + const first = await createSupervisor().run( + { + name: 'root', + act: async (_task, scope) => { + firstDeadline = scope.budget.deadlineMs + return 'first' + }, + }, + 'task', + { + budget, + rootIdentity, + runId: 'delayed-begin-deadline', + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + now: () => nowMs, + }, + ) + expect(first.kind).toBe('winner') + expect(firstDeadline).toBe(4_100) + + nowMs = 4_090 + let resumedDeadline = 0 + const resumed = await createSupervisor().run( + { + name: 'root', + act: async (_task, scope) => { + resumedDeadline = scope.budget.deadlineMs + return 'resumed' + }, + }, + 'task', + { + budget, + rootIdentity, + runId: 'delayed-begin-deadline', + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + resume: true, + now: () => nowMs, + }, + ) + + expect(resumed.kind).toBe('winner') + expect(resumedDeadline).toBe(firstDeadline) + expect(resumedDeadline).toBe(4_100) + }) + + it('does not settle a nested driver or refund its allocation while a descendant is live', async () => { + const baseJournal = new InMemorySpawnJournal() + const records: Array<{ root: string; event: SpawnEvent }> = [] + const journal: SpawnJournal = { + loadTree: (root) => baseJournal.loadTree(root), + beginTree: (root, at) => baseJournal.beginTree(root, at), + async appendEvent(root, event): Promise { + await baseJournal.appendEvent(root, event) + records.push({ root, event }) + }, + } + const runId = 'nested-join-barrier' + const activity = { live: 0, peak: 0 } + const descendantStarted = deferred() + const descendantExited = deferred() + let releaseDescendant: (() => void) | undefined + + const descendant = leafFromExecutor('descendant', () => { + const artifact: ExecutorResult = { + outRef: 'internal:descendant', + out: 'descendant result', + spent: zeroSpend, + } + return { + runtime: 'router', + execute(_task, signal): Promise> { + enter(activity) + descendantStarted.resolve() + return new Promise((resolve) => { + let finished = false + const finish = () => { + if (finished) return + finished = true + signal.removeEventListener('abort', finish) + leave(activity) + descendantExited.resolve() + resolve(artifact) + } + releaseDescendant = finish + if (signal.aborted) finish() + else signal.addEventListener('abort', finish, { once: true }) + }) + }, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => artifact, + } + }) + const nestedDriver: Agent = { + name: 'nested-driver', + async act(task, scope): Promise { + const spawned = scope.spawn(descendant, task, { + budget: { maxIterations: 1, maxTokens: 100 }, + label: 'descendant', + }) + expect(spawned.ok).toBe(true) + await descendantStarted.promise + return 'manager result' + }, + } + const second = trackedImmediateLeaf('second', activity) + const root: Agent = { + name: 'root', + async act(task, scope): Promise { + const manager = scope.spawn( + driverChild( + { name: 'manager', harness: 'cli-base', metadata: { role: 'driver' } }, + nestedDriver, + journal, + ), + task, + { + budget: { maxIterations: 1, maxTokens: 100 }, + label: 'manager', + }, + ) + expect(manager.ok).toBe(true) + const managerSettled = await scope.next() + const activeAfterManager = activity.live + const nestedTerminalBeforeManager = records.some( + ({ root: tree, event }) => tree !== runId && event.kind === 'settled', + ) + const secondSpawn = scope.spawn(second, task, { + budget: { maxIterations: 1, maxTokens: 100 }, + label: 'second', + }) + const secondSettled = secondSpawn.ok ? await scope.next() : null + return { + managerKind: managerSettled?.kind, + activeAfterManager, + nestedTerminalBeforeManager, + secondAccepted: secondSpawn.ok, + secondKind: secondSettled?.kind, + } + }, + } + + const result = await createSupervisor().run(root, 'task', { + budget: { maxIterations: 1, maxTokens: 100 }, + maxLiveWorkers: 2, + maxDepth: 3, + runId, + journal, + blobs: new InMemoryResultBlobStore(), + executors: withDriverExecutor(createExecutorRegistry()), + }) + const liveAtReturn = activity.live + releaseDescendant?.() + await descendantExited.promise + + expect(result.kind).toBe('winner') + if (result.kind !== 'winner') return + expect(result.out).toEqual({ + managerKind: 'done', + activeAfterManager: 0, + nestedTerminalBeforeManager: true, + secondAccepted: true, + secondKind: 'done', + }) + expect(activity.peak).toBe(1) + expect(liveAtReturn).toBe(0) + expect(result.tree.inFlight).toBe(0) + }) + + it('settles after its deadline even when an executor ignores AbortSignal', async () => { + let teardownCalls = 0 + const ignoring = leafFromExecutor('ignores-abort', () => ({ + runtime: 'router', + execute: () => new Promise>(() => {}), + async teardown(): Promise<{ destroyed: boolean }> { + teardownCalls += 1 + return { destroyed: true } + }, + resultArtifact(): ExecutorResult { + throw new Error('an executor that never settled has no result artifact') + }, + })) + const root: Agent = { + name: 'deadline-root', + async act(task, scope): Promise { + const spawned = scope.spawn(ignoring, task, { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'ignores-abort', + }) + expect(spawned.ok).toBe(true) + const settled = await scope.next() + if (settled?.kind === 'down') throw new Error(settled.reason) + return settled?.out ?? 'missing result' + }, + } + + const running = createSupervisor().run(root, 'task', { + budget: { maxIterations: 1, maxTokens: 10, deadlineMs: 15 }, + runId: 'ignores-abort-deadline', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }) + const outcome = await settleWithin(running, 250) + + expect(outcome.settled).toBe(true) + if (!outcome.settled) return + expect(outcome.value).toMatchObject({ + kind: 'no-winner', + reason: 'budget-exhausted', + tree: { inFlight: 0 }, + }) + expect(teardownCalls).toBe(1) + }) + + it('settles after its deadline even when executor teardown never acknowledges', async () => { + let teardownCalls = 0 + const ignoring = leafFromExecutor('ignores-teardown', () => ({ + runtime: 'router', + execute: () => new Promise>(() => {}), + teardown(): Promise<{ destroyed: boolean }> { + teardownCalls += 1 + return new Promise(() => {}) + }, + resultArtifact(): ExecutorResult { + throw new Error('an executor that never settled has no result artifact') + }, + })) + const root: Agent = { + name: 'teardown-deadline-root', + async act(task, scope): Promise { + const spawned = scope.spawn(ignoring, task, { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'ignores-teardown', + }) + expect(spawned.ok).toBe(true) + const settled = await scope.next() + if (settled?.kind === 'down') throw new Error(settled.reason) + return settled?.out ?? 'missing result' + }, + } + + const running = createSupervisor().run(root, 'task', { + budget: { maxIterations: 1, maxTokens: 10, deadlineMs: 15 }, + runId: 'ignores-teardown-deadline', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }) + const outcome = await settleWithin(running, 750) + + expect(outcome.settled).toBe(true) + if (!outcome.settled) return + expect(outcome.value).toMatchObject({ + kind: 'no-winner', + reason: 'budget-exhausted', + tree: { inFlight: 0 }, + }) + expect(teardownCalls).toBe(1) + }) + + it('turns unknown accounting from a crashing nested driver into a terminal down node', async () => { + const journal = new InMemorySpawnJournal() + const metered = deferred() + const nestedDriver: Agent = { + name: 'unknown-accounting-driver', + async act(_task, scope): Promise { + try { + await scope.meter({ + iterations: 1, + tokens: { input: 0, output: 0 }, + tokensKnown: false, + usd: 0, + ms: 1, + }) + } finally { + metered.resolve() + } + throw new Error('nested driver crashed') + }, + } + const root: Agent = { + name: 'root', + async act(task, scope): Promise { + const spawned = scope.spawn( + driverChild( + { name: 'unknown-manager', harness: 'cli-base', metadata: { role: 'driver' } }, + nestedDriver, + journal, + ), + task, + { + budget: { maxIterations: 2, maxTokens: 10 }, + label: 'unknown-manager', + }, + ) + expect(spawned.ok).toBe(true) + await metered.promise + return 'root result' + }, + } + const runId = 'unknown-nested-accounting' + + const result = await createSupervisor().run(root, 'task', { + budget: { maxIterations: 2, maxTokens: 10 }, + maxLiveWorkers: 1, + maxDepth: 2, + runId, + journal, + blobs: new InMemoryResultBlobStore(), + executors: withDriverExecutor(createExecutorRegistry()), + }) + const events = (await journal.loadTree(runId)) ?? [] + const manager = result.tree.nodes.find((node) => node.label === 'unknown-manager') + + expect(result.kind).toBe('winner') + expect(result.tree.inFlight).toBe(0) + expect(manager?.status).toBe('failed') + expect( + events.some( + (event) => event.kind === 'settled' && event.id === manager?.id && event.status === 'down', + ), + ).toBe(true) + expect( + events.some( + (event) => + event.kind === 'metered' && event.id === manager?.id && event.spend.tokensKnown === false, + ), + ).toBe(true) + expect(result.spentTotal.tokensKnown).toBe(false) + }) + + it('fails closed after a streaming provider crashes without terminal accounting', async () => { + let replacementExecutions = 0 + const crashed = leafFromExecutor('crashed-stream', () => ({ + runtime: 'router', + async *execute(): AsyncIterable { + yield { kind: 'iteration' } + yield { kind: 'tokens', input: 1, output: 0 } + yield { kind: 'cost', usd: 0.1 } + throw new Error('network died') + }, + teardown: async () => ({ destroyed: true }), + resultArtifact(): ExecutorResult { + throw new Error('a crashed stream has no terminal artifact') + }, + })) + const replacement = leafFromExecutor('replacement', () => ({ + runtime: 'router', + async execute(): Promise> { + replacementExecutions += 1 + return { outRef: 'internal', out: 'replacement', spent: zeroSpend } + }, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => ({ outRef: 'internal', out: 'replacement', spent: zeroSpend }), + })) + + const result = await createSupervisor().run( + { + name: 'root', + async act(task, scope): Promise { + const first = scope.spawn(crashed, task, { + budget: { maxIterations: 1, maxTokens: 100, maxUsd: 1 }, + label: 'crashed-stream', + }) + expect(first.ok).toBe(true) + const down = await scope.next() + const second = scope.spawn(replacement, task, { + budget: { maxIterations: 1, maxTokens: 99, maxUsd: 1 }, + label: 'replacement', + }) + return { downKind: down?.kind, second } + }, + }, + 'task', + { + budget: { maxIterations: 2, maxTokens: 199, maxUsd: 2 }, + runId: 'stream-crash-unknown-accounting', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }, + ) + + expect(result.kind).toBe('winner') + if (result.kind !== 'winner') return + expect(result.out).toEqual({ + downKind: 'down', + second: { ok: false, reason: 'budget-exhausted' }, + }) + expect(replacementExecutions).toBe(0) + expect(result.spentTotal).toMatchObject({ + iterations: 1, + tokens: { input: 1, output: 0 }, + tokensKnown: false, + usd: 0.1, + usdKnown: false, + }) + }) + + it('refuses an unmetered executor before compute and refunds its reservation', async () => { + let exemptExecutions = 0 + let exemptTeardowns = 0 + let measuredExecutions = 0 + const exempt = leafFromExecutor('subscription-cli', () => ({ + runtime: 'cli', + budgetExempt: true, + async execute(): Promise> { + exemptExecutions += 1 + return { outRef: 'internal', out: 'unmeasured', spent: zeroSpend } + }, + async teardown(): Promise<{ destroyed: boolean }> { + exemptTeardowns += 1 + return { destroyed: true } + }, + resultArtifact(): ExecutorResult { + throw new Error('an unmetered executor must never produce a supervised artifact') + }, + })) + const measured = leafFromExecutor('measured', () => ({ + runtime: 'router', + async execute(): Promise> { + measuredExecutions += 1 + return { + outRef: 'internal', + out: 'measured', + spent: { ...zeroSpend, iterations: 1, tokens: { input: 1, output: 0 } }, + } + }, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => ({ + outRef: 'internal', + out: 'measured', + spent: { ...zeroSpend, iterations: 1, tokens: { input: 1, output: 0 } }, + }), + })) + + const result = await createSupervisor().run( + { + name: 'root', + async act(task, scope): Promise { + const first = scope.spawn(exempt, task, { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'subscription-cli', + }) + expect(first.ok).toBe(true) + const refused = await scope.next() + const second = scope.spawn(measured, task, { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'measured', + }) + expect(second.ok).toBe(true) + const accepted = await scope.next() + return { + refusal: refused?.kind === 'down' ? refused.reason : 'not refused', + accepted: accepted?.kind === 'done' ? accepted.out : 'not accepted', + } + }, + }, + 'task', + { + budget: { maxIterations: 1, maxTokens: 10 }, + runId: 'refuse-unmetered-executor', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }, + ) + + expect(result.kind).toBe('winner') + if (result.kind !== 'winner') return + expect(result.out).toEqual({ + refusal: + 'scope.spawn: runtime "cli" does not report usage and cannot execute inside a budgeted supervisor', + accepted: 'measured', + }) + expect(exemptExecutions).toBe(0) + expect(exemptTeardowns).toBe(1) + expect(measuredExecutions).toBe(1) + expect(result.spentTotal).toMatchObject({ + iterations: 1, + tokens: { input: 1, output: 0 }, + usd: 0, + }) + }) +}) + +function resultLeaf(name: string, spent: Spend): Agent { + return leafFromExecutor(name, () => { + const artifact: ExecutorResult = { + outRef: `internal:${name}`, + out: `${name} result`, + spent, + } + return { + runtime: 'router', + execute: async () => artifact, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => artifact, + } + }) +} + +function trackedImmediateLeaf( + name: string, + activity: { live: number; peak: number }, +): Agent { + return leafFromExecutor(name, () => { + const artifact: ExecutorResult = { + outRef: `internal:${name}`, + out: `${name} result`, + spent: zeroSpend, + } + return { + runtime: 'router', + async execute(): Promise> { + enter(activity) + try { + await Promise.resolve() + return artifact + } finally { + leave(activity) + } + }, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => artifact, + } + }) +} + +function leafFromExecutor( + name: string, + makeExecutor: () => Executor, +): Agent { + const executor = makeExecutor() + const spec: AgentSpec = { + profile: { name } as AgentProfile, + harness: null, + executor: executor as Executor, + } + return { name, act: async () => undefined as Out, executorSpec: spec } as Agent & { + executorSpec: AgentSpec + } +} + +function enter(activity: { live: number; peak: number }): void { + activity.live += 1 + activity.peak = Math.max(activity.peak, activity.live) +} + +function leave(activity: { live: number }): void { + activity.live -= 1 +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +function settleWithin( + promise: Promise, + timeoutMs: number, +): Promise<{ settled: true; value: T } | { settled: false }> { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => resolve({ settled: false }), timeoutMs) + void promise.then( + (value) => { + clearTimeout(timer) + resolve({ settled: true, value }) + }, + (error: unknown) => { + clearTimeout(timer) + reject(error) + }, + ) + }) +} diff --git a/tests/kernel/supervisor-agent.test.ts b/tests/kernel/supervisor-agent.test.ts index bc08be56..8bfaaa9b 100644 --- a/tests/kernel/supervisor-agent.test.ts +++ b/tests/kernel/supervisor-agent.test.ts @@ -3,7 +3,11 @@ import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' import { createSupervisor } from '../../src/runtime/supervise/supervisor' -import { type DriveHarness, supervisorAgent } from '../../src/runtime/supervise/supervisor-agent' +import { + type DriveHarness, + type ResolveSupervisorTools, + supervisorAgent, +} from '../../src/runtime/supervise/supervisor-agent' import type { Agent, AgentSpec, @@ -12,6 +16,7 @@ import type { ExecutorResult, UsageEvent, } from '../../src/runtime/supervise/types' +import type { ToolLoopChat } from '../../src/runtime/tool-loop' import { scriptedBrain } from './scripted-brain' const perWorker: Budget = { maxIterations: 4, maxTokens: 1000 } @@ -74,14 +79,18 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen const brain = scriptedBrain([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'worker' }, task: 'go' } }, + { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, { content: 'done' }, ]) const root = supervisorAgent( - { name: 'root', harness: null, systemPrompt: 'drive the worker' }, + { + name: 'root', + harness: 'cli-base', + prompt: { systemPrompt: 'drive the worker' }, + }, { brain, blobs, makeWorkerAgent: () => worker, perWorker, maxTurns: 8 }, ) const result = await runSupervisor(root, blobs, journal) @@ -102,7 +111,11 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen await jsonRpc(coordinationMcpUrl, 'tools/call', { name: 'stop', arguments: {} }) } const root = supervisorAgent( - { name: 'sup', harness: 'opencode', systemPrompt: 'delegate, do not solve' }, + { + name: 'sup', + harness: 'opencode', + prompt: { systemPrompt: 'delegate, do not solve' }, + }, { blobs, makeWorkerAgent: () => deliveringLeaf('w', { answer: 7 }), perWorker, driveHarness }, ) const result = await runSupervisor(root, blobs, journal) @@ -123,9 +136,230 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen const blobs = new InMemoryResultBlobStore() expect(() => supervisorAgent( - { name: 'root', harness: null }, + { name: 'root', harness: 'cli-base' }, { blobs, makeWorkerAgent: () => deliveringLeaf('w', {}), perWorker }, ), ).toThrow(/router/) }) + + it('binds the same node-scoped product tool to router and external managers with trusted context', async () => { + const identity = { + profileDigest: `sha256:${'a'.repeat(64)}`, + taskDigest: `sha256:${'b'.repeat(64)}`, + correlation: { campaign: 'campaign-7' }, + } as const + const nodeContext = { + runId: 'sup', + runNamespace: 'durable-run-namespace', + ownerId: 'owner-root', + depth: 0, + identity, + } + const calls: Array<{ raw: unknown; context: unknown }> = [] + const resolveSupervisorTools: ResolveSupervisorTools = async () => [ + { + name: 'read_product_evidence', + description: 'Read one product-owned evidence record', + inputSchema: { + type: 'object', + properties: { + key: { type: 'string' }, + runId: { type: 'string' }, + trustedContext: { type: 'object' }, + }, + required: ['key'], + }, + handler: async (raw, context) => { + calls.push({ raw, context }) + return { + key: (raw as { key?: unknown }).key, + suppliedRunId: (raw as { runId?: unknown }).runId, + trustedRunId: context.runId, + trustedNodeId: context.nodeId, + } + }, + }, + ] + const modelArguments = { + key: 'claim-1', + runId: 'model-forged-run', + trustedContext: { nodeId: 'model-forged-node' }, + } + + let routerDescriptor: unknown + let routerTurn = 0 + const brain: ToolLoopChat = async (_messages, tools) => { + routerDescriptor = tools.find((entry) => entry.function.name === 'read_product_evidence') + routerTurn += 1 + return routerTurn === 1 + ? { + toolCalls: [ + { + id: 'product-call', + name: 'read_product_evidence', + arguments: JSON.stringify(modelArguments), + }, + ], + } + : { content: 'done', toolCalls: [] } + } + const routerBlobs = new InMemoryResultBlobStore() + await runSupervisor( + supervisorAgent( + { name: 'router-manager', harness: 'cli-base' }, + { + brain, + blobs: routerBlobs, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + nodeContext, + resolveSupervisorTools, + }, + ), + routerBlobs, + new InMemorySpawnJournal(), + ) + + let externalDescriptor: unknown + let externalCall: unknown + const externalBlobs = new InMemoryResultBlobStore() + const driveHarness: DriveHarness = async ({ coordinationMcpUrl }) => { + const listed = (await jsonRpc(coordinationMcpUrl, 'tools/list', {})) as { + result?: { tools?: unknown[] } + } + externalDescriptor = listed.result?.tools?.find( + (entry) => + typeof entry === 'object' && + entry !== null && + (entry as { name?: unknown }).name === 'read_product_evidence', + ) + externalCall = await jsonRpc(coordinationMcpUrl, 'tools/call', { + name: 'read_product_evidence', + arguments: modelArguments, + }) + } + await runSupervisor( + supervisorAgent( + { name: 'external-manager', harness: 'opencode' }, + { + blobs: externalBlobs, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + driveHarness, + nodeContext, + resolveSupervisorTools, + }, + ), + externalBlobs, + new InMemorySpawnJournal(), + ) + + expect(routerDescriptor).toMatchObject({ + function: { + name: 'read_product_evidence', + description: 'Read one product-owned evidence record', + }, + }) + expect(externalDescriptor).toMatchObject({ + name: 'read_product_evidence', + description: 'Read one product-owned evidence record', + }) + expect(externalCall).toMatchObject({ + result: { + structuredContent: { + key: 'claim-1', + suppliedRunId: 'model-forged-run', + trustedRunId: 'sup', + trustedNodeId: 'sup', + }, + }, + }) + expect(calls).toHaveLength(2) + for (const call of calls) { + expect(call.raw).toEqual(modelArguments) + expect(Object.isFrozen(call.raw)).toBe(true) + expect(call.context).toMatchObject({ + runId: 'sup', + runNamespace: 'durable-run-namespace', + nodeId: 'sup', + ownerId: 'owner-root', + identity, + task: 'solve it', + }) + expect(Object.isFrozen(call.context)).toBe(true) + expect(Object.isFrozen((call.context as { identity: unknown }).identity)).toBe(true) + } + }) + + it('captures the resolver and rejects descriptor collisions before brain compute or MCP listen', async () => { + const seed = { + runId: 'sup', + runNamespace: 'namespace', + ownerId: 'owner', + depth: 0, + identity: { + profileDigest: `sha256:${'c'.repeat(64)}`, + taskDigest: `sha256:${'d'.repeat(64)}`, + }, + } as const + let originalCalls = 0 + let replacementCalls = 0 + let brainCalls = 0 + let harnessCalls = 0 + const deps = { + blobs: new InMemoryResultBlobStore(), + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + nodeContext: seed, + resolveSupervisorTools: (async () => { + originalCalls += 1 + return [ + { + name: 'spawn_agent', + description: 'collision', + inputSchema: { type: 'object' }, + handler: async () => ({}), + }, + ] + }) as ResolveSupervisorTools, + } + const brain: ToolLoopChat = async () => { + brainCalls += 1 + return { content: 'must not run', toolCalls: [] } + } + const mutableRouterDeps = { ...deps, brain } + const router = supervisorAgent({ name: 'router', harness: 'cli-base' }, mutableRouterDeps) + mutableRouterDeps.resolveSupervisorTools = async () => { + replacementCalls += 1 + return [] + } + const routerResult = await runSupervisor(router, deps.blobs, new InMemorySpawnJournal()) + expect(routerResult.kind).toBe('no-winner') + expect(originalCalls).toBe(1) + expect(replacementCalls).toBe(0) + expect(brainCalls).toBe(0) + + const externalBlobs = new InMemoryResultBlobStore() + const external = supervisorAgent( + { name: 'external', harness: 'opencode' }, + { + ...deps, + blobs: externalBlobs, + resolveSupervisorTools: async () => [ + { + name: 'spawn_agent', + description: 'collision', + inputSchema: { type: 'object' }, + handler: async () => ({}), + }, + ], + driveHarness: async () => { + harnessCalls += 1 + }, + }, + ) + const externalResult = await runSupervisor(external, externalBlobs, new InMemorySpawnJournal()) + expect(externalResult.kind).toBe('no-winner') + expect(harnessCalls).toBe(0) + }) }) diff --git a/tests/runtime/executor-config-snapshot.test.ts b/tests/runtime/executor-config-snapshot.test.ts new file mode 100644 index 00000000..16b32266 --- /dev/null +++ b/tests/runtime/executor-config-snapshot.test.ts @@ -0,0 +1,266 @@ +import { describe, expect, it } from 'vitest' +import type { + AgentEnvironmentProvider, + AgentEnvironmentProviderRegistry, +} from '../../src/runtime/environment-provider' +import { + bindReusableExecutorExecutionId, + captureReusableExecutorConfig, + createExecutor, + type ExecutorConfig, +} from '../../src/runtime/supervise/runtime' +import type { AgentSpec, ExecutorContext, Runtime } from '../../src/runtime/supervise/types' +import type { ExecCtx, SandboxClient } from '../../src/runtime/types' + +const spec: AgentSpec = { profile: { name: 'snapshot-worker' }, harness: null } +const context: ExecutorContext = { signal: new AbortController().signal, seams: {} } + +describe('createExecutor config intake', () => { + it('captures every backend variant while retaining only explicit live ports', () => { + const executeToolCall = async () => 'tool result' + const onToolStep = () => {} + const runGit = () => ({ stdout: '', stderr: '', exitCode: 0 }) + const runCommand = async () => ({ exitCode: 0, output: '' }) + const provider = { + name: 'live-provider', + capabilities: async () => { + throw new Error('not executed') + }, + create: async () => { + throw new Error('not executed') + }, + } as AgentEnvironmentProvider + const taskToTurn = () => ({ prompt: 'live mapper' }) + const sandboxClient = { + create: async () => { + throw new Error('not executed') + }, + } as SandboxClient + const hooks = { onEvent: () => {} } + const traceEmitter = { emit: () => {} } + const onSandboxEvent = () => {} + const runHandle = { observe: () => {} } as unknown as NonNullable + + const cases: Array<{ name: string; config: ExecutorConfig; runtime: Runtime }> = [ + { + name: 'router', + config: { + backend: 'router', + routerBaseUrl: 'http://router.test', + routerKey: 'key', + model: 'model', + }, + runtime: 'router', + }, + { + name: 'router-tools', + config: { + backend: 'router-tools', + routerBaseUrl: 'http://router.test', + routerKey: 'key', + model: 'model', + tools: [], + executeToolCall, + onToolStep, + }, + runtime: 'router', + }, + { + name: 'bridge', + config: { + backend: 'bridge', + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'model', + }, + runtime: 'cli', + }, + { + name: 'cli', + config: { backend: 'cli', bin: '/bin/true', args: ['--version'] }, + runtime: 'cli', + }, + { + name: 'cli-worktree', + config: { + backend: 'cli-worktree', + repoRoot: '/repo', + harness: 'codex', + runGit, + runCommand, + }, + runtime: 'cli', + }, + { + name: 'provider', + config: { + backend: 'provider', + provider, + runtime: 'provider-runtime', + taskToTurn, + defaults: { workspace: { cwd: '/repo' } }, + }, + runtime: 'provider-runtime', + }, + { + name: 'pi', + config: { backend: 'pi', bin: 'pi', args: ['--test'], model: 'provider/model' }, + runtime: 'pi', + }, + { + name: 'sandbox', + config: { + backend: 'sandbox', + harness: 'codex', + sandboxClient, + maxIterations: 1, + lineage: { sessionContinuity: true }, + steering: { maxTurns: 2 }, + loopCtx: { + hooks, + traceEmitter, + onSandboxEvent, + runHandle, + traceId: 'trace-id', + parentSpanId: 'parent-id', + }, + }, + runtime: 'sandbox', + }, + ] + + for (const testCase of cases) { + const originalBackend = testCase.config.backend + const factory = createExecutor(testCase.config) + const mutableConfig = testCase.config as { backend: string } + mutableConfig.backend = originalBackend === 'router' ? 'cli' : 'router' + + expect(factory(spec, context).runtime, testCase.name).toBe(testCase.runtime) + } + }) + + it('resolves a named provider once instead of retaining a mutable registry lookup', () => { + const providerA = { + name: 'provider-a', + capabilities: async () => { + throw new Error('not executed') + }, + create: async () => { + throw new Error('not executed') + }, + } as AgentEnvironmentProvider + const providerB = { ...providerA, name: 'provider-b' } as AgentEnvironmentProvider + let current = providerA + const registry = { + require: () => current, + } as AgentEnvironmentProviderRegistry + const factory = createExecutor({ + backend: 'provider', + provider: 'selected-provider', + registry, + }) + current = providerB + + expect(factory(spec, context).runtime).toBe('provider-a') + }) + + it('rejects reusable profile overlays and fixed execution ids', () => { + const invalid: ExecutorConfig[] = [ + { + backend: 'bridge', + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'model', + agentProfile: { name: 'late-overlay' }, + }, + { + backend: 'bridge', + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'model', + sessionId: 'SHARED', + }, + { + backend: 'cli-worktree', + repoRoot: '/repo', + harness: 'codex', + runId: 'SHARED', + }, + { + backend: 'cli-worktree', + repoRoot: '/repo', + bridge: { + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'model', + sessionId: 'SHARED', + }, + }, + ] + + for (const config of invalid) { + expect(() => captureReusableExecutorConfig(config, 'reusable-test')).toThrow( + /not allowed|isolated id/, + ) + } + }) + + it('keeps fixed ids available to explicitly single-execution factories', () => { + const direct: ExecutorConfig[] = [ + { + backend: 'bridge', + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'model', + sessionId: 'PINNED-ONCE', + }, + { + backend: 'cli-worktree', + repoRoot: '/repo', + harness: 'codex', + runId: 'PINNED-ONCE', + }, + ] + + for (const config of direct) { + const executor = createExecutor(config)(spec, context) + expect(executor.runtime).toBe('cli') + } + }) + + it('binds worktree and bridge backends to the supplied durable execution identity', () => { + const bridge = captureReusableExecutorConfig( + { + backend: 'bridge', + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'model', + }, + 'bridge-test', + ) + const worktree = captureReusableExecutorConfig( + { + backend: 'cli-worktree', + repoRoot: '/repo', + bridge: { + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'model', + }, + }, + 'worktree-test', + ) + + expect(bindReusableExecutorExecutionId(bridge, 'execution-a')).toMatchObject({ + backend: 'bridge', + sessionId: 'execution-a', + }) + expect(bindReusableExecutorExecutionId(worktree, 'execution-a')).toMatchObject({ + backend: 'cli-worktree', + runId: 'execution-a', + }) + expect(bindReusableExecutorExecutionId(worktree, 'execution-b')).toMatchObject({ + runId: 'execution-b', + }) + }) +}) From ed31aaf7d1b618222a494f3af26a741697a832d8 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 05:29:01 -0600 Subject: [PATCH 02/12] feat(supervise): attest profile materialization --- docs/canonical-api.md | 17 +- src/durable/jsonl-file.ts | 71 ++++ src/mcp/worktree-harness.ts | 126 +++--- src/runtime/personify/trajectory.ts | 12 +- src/runtime/supervise/authoring.ts | 136 ++++--- src/runtime/supervise/completion-gate.ts | 4 +- src/runtime/supervise/coordination-log.ts | 168 ++++++-- src/runtime/supervise/materialization.ts | 361 ++++++++++++++++++ src/runtime/supervise/pi-executor.ts | 209 +++++++--- src/runtime/supervise/runtime.ts | 2 - .../supervise/worktree-cli-executor.ts | 218 +++++++---- tests/kernel/materialization-evidence.test.ts | 268 +++++++++++++ .../spawn-journal-replay-identity.test.ts | 173 +++++++++ tests/runtime/worktree-cli-executor.test.ts | 81 +++- 14 files changed, 1562 insertions(+), 284 deletions(-) create mode 100644 src/durable/jsonl-file.ts create mode 100644 src/runtime/supervise/materialization.ts create mode 100644 tests/kernel/materialization-evidence.test.ts create mode 100644 tests/runtime/spawn-journal-replay-identity.test.ts diff --git a/docs/canonical-api.md b/docs/canonical-api.md index b9cf8943..d6aaf8ab 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -53,6 +53,21 @@ Skills remain separate resources that the runtime can invoke; do not concatenate **Therefore the supervisor's only intelligence is AUTHORING full profiles**: the optimizable self-improvement surface: read the task, decompose it, and for each sub-task author the *complete* profile (which prompt, skills, tools/MCP, hooks, subagents, model). The quality of a worker IS the quality of the profile authored for it. **The harness executes; you compose.** +`supervise(profile, task, { backend, ... })` keeps that contract at execution time. +It validates and freezes each authored child as a canonical `AgentProfile`, applies the shared profile-security policy and optional product authorization before reserving compute, and sends the exact authorized profile to the selected backend. +A child with `metadata.role: 'driver'` becomes another supervisor over the same budget and may author its own children; every other child is a leaf. +The parent reserves that driver's allocation once; the nested manager partitions only that allocation, so nested work is neither double-charged nor allowed to borrow outside its branch. +For an external-harness supervisor, the automatic path is local CLI bridge execution: Runtime adds the live coordination MCP under the reserved `agent-runtime-coordination` alias and executes the rest of the profile unchanged through a `bridge` `driverBackend ?? backend`. +Before that runtime starts, the spawn journal records a `materialized` receipt containing the authored-profile, effective-profile, and platform-attachment digests. +Volatile endpoints, credentials, and process/session bindings are never stored there. +Each execution or restart instead appends a digest-only `execution-bound` receipt for that attempt, so the stable profile identity survives restarts while every concrete attempt remains distinguishable. +A remote sandbox cannot reach Runtime's loopback coordination server automatically, so it requires an explicit `driveHarness` that supplies its own reachable relay or tunnel. +For manager-authored child profiles, the default security policy blocks local MCP processes, every remote MCP host, shell hooks, and ambient connection grants; products opt in by passing an explicit `profileSecurity` policy and remote-host allowlist. + +The in-process router supervisor is deliberately narrower because it has no harness environment to materialize. +It reads the profile's prompt and model, while its callable work tools remain explicit `extraTools`; use an external backend when tools, resources, hooks, subagents, permissions, or modes must execute. +Supplying `makeWorkerAgent` is the advanced caller-owned path, so Runtime cannot apply backend-derived profile security, spawn authorization, or recursive-driver selection around it; `authorizeMessage` still governs continuations sent through Runtime's coordination tools. + ## 2. Decision table: "I want to ___ → use ___ → NOT ___" This table is judgment-only: it maps an intent to the ONE primitive to reach for and the thing NOT to build. It is not an inventory: **for the full list of what exists (every export, its import path, its one-line summary) see the generated `docs/api/primitive-catalog.md`; for full signatures, the per-module `docs/api/` pages.** Each row tags its import subpath; a row is a LOCAL export of this package unless tagged with a substrate package (`agent-eval/contract`, `agent-eval/campaign`, `@tangle-network/sandbox`) or `bench`. @@ -128,7 +143,7 @@ A general "loop" primitive is the single most common modelling error in this rep | State any benchmark/A-B claim | `pairedLift(...)` (bench) over `pairedBootstrap`/`heldoutSignificance` (substrate) | your own bootstrap loop/PRNG per gate; a point lift without `low/high/pairs` | | Let an agent **delegate ONE generic INTENT** (no fixed coder/researcher type) and get the result + real spend SYNCHRONOUSLY | the **`delegate` tool**: `createDelegateHandler` via `createMcpServer({ delegateSupervisor })`; mount it over the `agent-runtime mcp` bin with `MCP_ENABLE_DELEGATE=1` (the bin authors a supervisor over a `sandbox` backend): `/mcp` | a hardcoded coder/researcher profile, or task-specific `delegate_code`/`delegate_research` verbs (RETIRED): `delegate` is the ONE delegation path and the only one with a cost channel | | Run a coding task INSIDE the agent's OWN sandbox session (a sibling box, fresh branch, validated patch) | `detachedSessionDelegate({ sandboxClient \| executor, workerProfile? })`: `/mcp` (pass the worker `AgentProfile`; omit for a minimal model-only default) | a hardcoded coder profile baked into the delegate; `delegate()` (that spawns workers in a *chosen* backend, not the agent's own session) | -| Have a **supervisor spawn + live-drive workers in a backend you choose** and observe or steer them while the coordinator is alive | the **coordination MCP** via `createCoordinationTools` / `serveCoordinationMcp` over a live `Scope`; each worker's leaf is `createExecutor({ backend })` | `detachedSessionDelegate`, which is own-sandbox-session only and one-shot. Supervised-tree restart recovery is not implemented. | +| Have a **supervisor spawn + live-drive workers in a backend you choose** and observe or steer them while the coordinator is alive | `supervise(profile, task, { backend, runDir, runId })` for the complete path, or the lower-level **coordination MCP** via `createCoordinationTools` / `serveCoordinationMcp` over a live `Scope`; same-host restart restores committed work, exact identities, spend, waits, and the original deadline while preserving prior coordination evidence | `detachedSessionDelegate`, which is own-sandbox-session only and one-shot; same-host restart does not reattach work that was active when its process died, and cross-box durable coordination still requires a transport binding | | Stand up a vertical agent in the eval loop | `defineAgent(manifest)` + `createSurfaceImprovementProposer`: `/agent` | a per-vertical manifest parser, surface-validator, or bespoke findings-to-patch mapper | | Observe + deliver Intelligence on a live agent (send RunRecords + receive certified profile/diffs) | `withIntelligence(agent, { project, target })`: `/intelligence` (proposals surfaced, never auto-applied; `effort: 'off'` proves inference-only billing) | a custom trace-wrapper, a second receive path, or hand-rolled effort/tier config | | Turn trace evidence into one measured, review-only agent proposal | `proposeAgentImprovement({ analysis, profile, improvement, buildExperiment, placeCell })` in `/intelligence` (Runtime seals the optimizer ancestry) | manually joining analysis, optimizer ancestry, exact candidate execution, uncertainty, and candidate identity | diff --git a/src/durable/jsonl-file.ts b/src/durable/jsonl-file.ts new file mode 100644 index 00000000..c0cedb73 --- /dev/null +++ b/src/durable/jsonl-file.ts @@ -0,0 +1,71 @@ +import type { FileHandle } from 'node:fs/promises' + +/** Parse an append-only JSONL file without treating a torn final write as committed data. + * A malformed newline-terminated or non-final record is corruption and fails loud. */ +export function parseCommittedJsonLines(text: string, source: string): T[] { + const lines = text.split('\n') + const finalIndex = lines.length - 1 + const records: T[] = [] + + for (const [index, line] of lines.entries()) { + if (line.length === 0) continue + try { + records.push(JSON.parse(line) as T) + } catch (cause) { + const isInvalidUnterminatedTail = index === finalIndex && !text.endsWith('\n') + if (isInvalidUnterminatedTail) break + throw new Error(`${source}: malformed JSONL record at line ${index + 1}`, { cause }) + } + } + + return records +} + +/** FileHandle.write may legally make a short write. Loop until every byte is appended. */ +export async function writeAllBytes( + handle: Pick, + value: string | Uint8Array, +): Promise { + const bytes = typeof value === 'string' ? Buffer.from(value) : Buffer.from(value) + let offset = 0 + while (offset < bytes.byteLength) { + const { bytesWritten } = await handle.write(bytes, offset, bytes.byteLength - offset, null) + if (bytesWritten <= 0) { + throw new Error(`append-only file write made no progress at byte ${offset}`) + } + offset += bytesWritten + } +} + +/** Prepare a recovered JSONL file for its next append. A valid unterminated final value is retained + * and needs a separator; an invalid unterminated tail was never committed and is truncated. */ +export async function prepareJsonlAppend(path: string): Promise { + const fs = await import('node:fs/promises') + let bytes: Buffer + try { + bytes = await fs.readFile(path) + } catch (error) { + if (isNoEntError(error)) return false + throw error + } + if (bytes.byteLength === 0 || bytes[bytes.byteLength - 1] === 0x0a) return false + + const lastNewline = bytes.lastIndexOf(0x0a) + const tail = bytes.subarray(lastNewline + 1).toString('utf8') + try { + JSON.parse(tail) + return true + } catch { + await fs.truncate(path, lastNewline + 1) + return false + } +} + +function isNoEntError(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code: unknown }).code === 'ENOENT' + ) +} diff --git a/src/mcp/worktree-harness.ts b/src/mcp/worktree-harness.ts index 062c8087..bab5a446 100644 --- a/src/mcp/worktree-harness.ts +++ b/src/mcp/worktree-harness.ts @@ -25,6 +25,7 @@ import type { AgentProfile } from '@tangle-network/agent-interface' import { applyWorkspacePlan, type HarnessId, + hashWorkspacePlan, materializeProfile, type WorkspacePlan, type WorkspacePlanReceipt, @@ -78,6 +79,13 @@ export interface WorktreeProfileMaterializationReceipt { } } +/** Pure, pre-execution identity for the exact profile workspace plan this module later applies. */ +export interface WorktreeProfileExecutionPlan { + readonly workspacePlanDigest: string + readonly harness: HarnessId + readonly resourceInstructions: WorktreeProfileMaterializationReceipt['resourceInstructions'] +} + /** The canonical result of one worktree-harness run, projected by each port to its own shape. */ export interface WorktreeHarnessResult { /** The branch the worktree was cut on (`delegate/`). */ @@ -209,6 +217,11 @@ export async function runWorktreeHarness( const checkTimeoutMs = opts.checkTimeoutMs ?? opts.harnessTimeoutMs ?? 5 * 60 * 1000 const cap = opts.checkOutputCap ?? defaultCheckOutputCap + // Profile support is a pure admission decision. Resolve it before creating a branch/worktree so + // a refused candidate leaves no repository state behind and starts no external process. + const prepared = prepareWorktreeProfile(opts.profile, opts.harness) + const { plan, resourceInstructions } = prepared + const worktree = await createWorktree({ repoRoot: opts.repoRoot, runId: opts.runId, @@ -224,19 +237,6 @@ export async function runWorktreeHarness( }) try { - assertSupportedWorktreeProfile(opts.profile, opts.harness) - assertSafeProfileResourcePaths(opts.profile) - const resourceInstructions = resolveResourceInstructions(opts.profile) - const workspaceProfile = materializationOnlyProfile(opts.profile) - const plan = materializeProfile(workspaceProfile, materializerHarness(opts.harness)) - if (plan.unsupported.length > 0) { - throw new Error( - `runWorktreeHarness: profile cannot be materialized for ${opts.harness}: ${plan.unsupported - .map(({ dimension, reason }) => `${dimension}: ${reason}`) - .join('; ')}`, - ) - } - assertSafeMaterializedPaths(plan) const applied = applyWorkspacePlan(plan, worktree.path, { existingFiles: 'reject' }) if (applied.unsupported.length > 0) { throw new Error('runWorktreeHarness: applied profile unexpectedly retained unsupported rows') @@ -335,6 +335,42 @@ export async function runWorktreeHarness( } } +/** Compute the same plan identity `runWorktreeHarness` consumes, with no repository or process IO. */ +export function worktreeProfileExecutionPlan( + profile: AgentProfile, + harness: LocalHarness, +): WorktreeProfileExecutionPlan { + const { plan, resourceInstructions } = prepareWorktreeProfile(profile, harness) + return Object.freeze({ + workspacePlanDigest: hashWorkspacePlan(plan), + harness: plan.harness, + resourceInstructions: resourceInstructionReceipt(resourceInstructions), + }) +} + +function prepareWorktreeProfile( + profile: AgentProfile, + harness: LocalHarness, +): { + readonly plan: WorkspacePlan + readonly resourceInstructions: string | undefined +} { + const resourceInstructions = resolveResourceInstructions(profile) + const workspaceProfile = materializationOnlyProfile(profile) + assertSupportedWorktreeProfile(profile, harness) + assertSafeProfileResourcePaths(profile) + const plan = materializeProfile(workspaceProfile, materializerHarness(harness)) + if (plan.unsupported.length > 0) { + throw new Error( + `runWorktreeHarness: profile cannot be materialized for ${harness}: ${plan.unsupported + .map(({ dimension, reason }) => `${dimension}: ${reason}`) + .join('; ')}`, + ) + } + assertSafeMaterializedPaths(plan) + return { plan, resourceInstructions } +} + function resolveResourceInstructions(profile: AgentProfile): string | undefined { const instructions = profile.resources?.instructions if (instructions === undefined) return undefined @@ -393,46 +429,12 @@ function assertSupportedWorktreeProfile(profile: AgentProfile, harness: LocalHar // `profile.harness` is only a preference and the explicit run option wins. Model small/provider/ // metadata fields are routing or descriptive hints; this fixed one-shot path only selects the // concrete `model.default`. `resources.failOnError` never weakens this path's fail-closed policy. - const unsupportedAxes = [ - hasEntries(profile.tools) ? 'tools' : null, - hasEntries(profile.permissions) ? 'permissions' : null, - profile.connections && profile.connections.length > 0 ? 'connections' : null, - hasEntries(profile.confidential) ? 'confidential' : null, - hasEntries(profile.modes) ? 'modes' : null, - hasEntries(profile.extensions) ? 'extensions' : null, - ].filter((axis): axis is string => axis !== null) + const unsupportedAxes = [hasEntries(profile.extensions) ? 'extensions' : null].filter( + (axis): axis is string => axis !== null, + ) if (profile.model?.reasoningEffort !== undefined && harness !== 'codex') { unsupportedAxes.push('model.reasoningEffort') } - for (const [name, server] of Object.entries(profile.mcp ?? {})) { - const path = `mcp[${JSON.stringify(name)}]` - if (server.enabled === false && harness !== 'opencode') { - unsupportedAxes.push(`${path}.enabled`) - } - if (hasEntries(server.headers) && harness === 'codex') { - unsupportedAxes.push(`${path}.headers`) - } - if (server.cwd !== undefined && harness === 'opencode') { - unsupportedAxes.push(`${path}.cwd`) - } - } - if (harness === 'claude') { - for (const [event, commands] of Object.entries(profile.hooks ?? {})) { - for (const [index, command] of commands.entries()) { - const path = `hooks[${JSON.stringify(event)}][${index}]` - if (hasEntries(command.env)) unsupportedAxes.push(`${path}.env`) - if (command.blocking !== undefined) unsupportedAxes.push(`${path}.blocking`) - } - } - } - for (const [name, subagent] of Object.entries(profile.subagents ?? {})) { - const path = `subagents[${JSON.stringify(name)}]` - if (hasEntries(subagent.permissions)) unsupportedAxes.push(`${path}.permissions`) - if (subagent.maxSteps !== undefined) unsupportedAxes.push(`${path}.maxSteps`) - if (hasEntries(subagent.tools) && harness !== 'claude') { - unsupportedAxes.push(`${path}.tools`) - } - } if (unsupportedAxes.length > 0) { throw new Error( `runWorktreeHarness: profile requests unsupported worktree behavior: ${unsupportedAxes.join(', ')}`, @@ -473,21 +475,27 @@ function profileMaterializationReceipt( applied: WorkspacePlanReceipt, resourceInstructions: string | undefined, ): WorktreeProfileMaterializationReceipt { - const instructionBytes = - resourceInstructions === undefined ? null : Buffer.from(resourceInstructions, 'utf8') return { workspacePlanDigest: applied.workspacePlanDigest, writtenPaths: [...applied.written], unsupported: [...applied.unsupported], environmentNames: Object.keys(applied.env).sort(), flags: [...applied.flags], - resourceInstructions: instructionBytes - ? { - delivery: 'invocation-prompt', - sha256: `sha256:${createHash('sha256').update(instructionBytes).digest('hex')}`, - byteLength: instructionBytes.byteLength, - } - : { delivery: 'none', sha256: null, byteLength: 0 }, + resourceInstructions: resourceInstructionReceipt(resourceInstructions), + } +} + +function resourceInstructionReceipt( + resourceInstructions: string | undefined, +): WorktreeProfileMaterializationReceipt['resourceInstructions'] { + if (resourceInstructions === undefined) { + return { delivery: 'none', sha256: null, byteLength: 0 } + } + const instructionBytes = Buffer.from(resourceInstructions, 'utf8') + return { + delivery: 'invocation-prompt', + sha256: `sha256:${createHash('sha256').update(instructionBytes).digest('hex')}`, + byteLength: instructionBytes.byteLength, } } diff --git a/src/runtime/personify/trajectory.ts b/src/runtime/personify/trajectory.ts index 97a2b839..7690e3cd 100644 --- a/src/runtime/personify/trajectory.ts +++ b/src/runtime/personify/trajectory.ts @@ -69,7 +69,14 @@ export async function trajectoryReport( // they accumulate ONTO the settled child-work base regardless of seq order; closes are the // settlements/cancellations that set node status. const closes = events - .filter((ev) => ev.kind !== 'spawned' && ev.kind !== 'waiting' && ev.kind !== 'metered') + .filter( + (ev) => + ev.kind !== 'spawned' && + ev.kind !== 'waiting' && + ev.kind !== 'metered' && + ev.kind !== 'materialized' && + ev.kind !== 'execution-bound', + ) .sort(bySeq) const nodes = new Map() @@ -285,6 +292,7 @@ function addNodeSpend(a: Spend, b: Spend): Spend { return { iterations: a.iterations + b.iterations, tokens: { input: a.tokens.input + b.tokens.input, output: a.tokens.output + b.tokens.output }, + ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), usd: a.usd + b.usd, ...(a.usdKnown === false || b.usdKnown === false ? { usdKnown: false } : {}), ms: a.ms + b.ms, @@ -295,6 +303,7 @@ function cloneSpend(spend: Spend): Spend { return { iterations: spend.iterations, tokens: { input: spend.tokens.input, output: spend.tokens.output }, + ...(spend.tokensKnown === false ? { tokensKnown: false } : {}), usd: spend.usd, ...(spend.usdKnown === false ? { usdKnown: false } : {}), ms: spend.ms, @@ -305,6 +314,7 @@ function cloneSpend(spend: Spend): Spend { function addSpend(acc: Spend, delta: Spend): void { acc.iterations += delta.iterations addTokenUsage(acc.tokens, delta.tokens) + if (delta.tokensKnown === false) acc.tokensKnown = false acc.usd += delta.usd if (delta.usdKnown === false) acc.usdKnown = false acc.ms += delta.ms diff --git a/src/runtime/supervise/authoring.ts b/src/runtime/supervise/authoring.ts index c8ee56ed..be56c3de 100644 --- a/src/runtime/supervise/authoring.ts +++ b/src/runtime/supervise/authoring.ts @@ -15,30 +15,38 @@ */ import { type AnalystFinding, computeFindingId, makeFinding } from '@tangle-network/agent-eval' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { + type AgentProfile, + type AgentProfilePrompt, + agentProfileSchema, +} from '@tangle-network/agent-interface' import { contentAddress } from '../../durable/spawn-journal' import { type RouterConfig, routerChatWithUsage } from '../router-client' import { type DeliverableSpec, gateOnDeliverable } from './completion-gate' +import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' import type { Agent, AgentSpec, Executor, ExecutorResult } from './types' -/** What the supervisor AUTHORS per sub-task — a worker recipe (a partial `AgentProfile`). */ -export interface AuthoredProfile { - name: string - /** The rich, task-specific instructions the supervisor wrote for THIS worker. */ - systemPrompt: string - /** The model the supervisor chose for this sub-task (falls back to the run default). */ - model?: string +/** What the supervisor AUTHORS per sub-task: one complete canonical profile whose name and + * task-specific system prompt are present. Every other `AgentProfile` axis is preserved exactly. */ +export type AuthoredProfile = AgentProfile & { + readonly name: string + readonly prompt: AgentProfilePrompt & { readonly systemPrompt: string } } /** Narrow an untyped `spawn_agent` profile argument to an `AuthoredProfile`, or null if the * supervisor failed to author one (empty/placeholder profile — a skill violation worth catching). */ export function asAuthoredProfile(raw: unknown): AuthoredProfile | null { - const p = raw as Partial | undefined - if (!p || typeof p.systemPrompt !== 'string' || p.systemPrompt.trim().length === 0) return null + const parsed = agentProfileSchema.safeParse(raw) + if (!parsed.success) return null + const systemPrompt = parsed.data.prompt?.systemPrompt + if (typeof systemPrompt !== 'string' || systemPrompt.trim().length === 0) return null return { - name: typeof p.name === 'string' && p.name.length > 0 ? p.name : 'worker', - systemPrompt: p.systemPrompt, - ...(typeof p.model === 'string' ? { model: p.model } : {}), + ...parsed.data, + name: + typeof parsed.data.name === 'string' && parsed.data.name.length > 0 + ? parsed.data.name + : 'worker', + prompt: { ...parsed.data.prompt, systemPrompt }, } } @@ -51,19 +59,21 @@ export function supervisorInstructions(opts?: { goal?: string }): string { 'For the task you are given:', '1. DECOMPOSE it into the smallest set of sub-tasks a single focused worker can each deliver.', '2. For EACH sub-task, AUTHOR a worker by calling spawn_agent with a COMPLETE `profile`:', - ' • name: a short id for the worker.', - ' • systemPrompt: rich, specific instructions for THIS sub-task — tell the worker exactly what to produce, how to use its tools fully, and what "done" means. Never a one-liner; write the prompt a power-user would write.', - ' • model: the model best suited to this sub-task (omit to use the default).', + ' • name and description: who this specialist is and why it exists.', + ' • prompt.systemPrompt: rich instructions for THIS sub-task — exact output, process, evidence, and what "done" means.', + ' • model.default, model.reasoningEffort, and harness: choose the execution system deliberately when the task benefits from it.', + ' • tools, mcp, resources.skills/files/instructions, hooks, subagents, permissions, and modes: grant or attach every capability the worker needs; omit an axis only when it is intentionally unnecessary.', + ' • metadata.role="driver" when this child should be a sub-supervisor that may author and drive its own children.', ' NEVER spawn a worker with an empty profile. The quality of the worker IS the quality of the profile you write.', "3. await_event (kinds:['settled']) to collect each worker. Its result says valid:true only if the deployable check passed.", - '4. If a worker did NOT deliver, AUTHOR A NEW worker whose systemPrompt names the SPECIFIC failure and how to fix it — never just retry the same prompt.', + '4. If a worker did NOT deliver, AUTHOR A NEW profile whose prompt.systemPrompt names the SPECIFIC failure and how to fix it — never just retry the same profile.', '5. Stop (reply with no tool call) once the work is delivered. You cannot declare done yourself — only a delivered (valid:true) worker counts.', ...(opts?.goal ? ['', `The goal: ${opts.goal}`] : []), ].join('\n') } -/** Build a worker AGENT from a profile the supervisor authored: the authored `systemPrompt` + - * `model` shape the worker's one model call; the deliverable gates settlement (valid ⟺ delivered). */ +/** Build a router-only worker from an authored profile. This helper executes the prompt/model axes; + * use `workerFromBackend` for full materialization of tools, MCP, resources, hooks, and subagents. */ export function authoredWorker( profile: AuthoredProfile, opts: { @@ -73,42 +83,66 @@ export function authoredWorker( temperature?: number }, ): Agent { - let artifact: ExecutorResult | undefined - const model = profile.model ?? opts.cfg.model - const inner: Executor = { - runtime: 'router', - async execute(_t, signal) { - const res = await routerChatWithUsage( - { ...opts.cfg, model }, - [ - { role: 'system', content: profile.systemPrompt }, - { role: 'user', content: opts.taskPrompt }, - ], - { temperature: opts.temperature ?? 0.4, ...(signal ? { signal } : {}) }, - ) - artifact = { - outRef: contentAddress(res.content), - out: res.content, - spent: { - iterations: 1, - tokens: res.usage ?? { input: 0, output: 0 }, - usd: res.costUsd ?? 0, - ms: 0, + const model = profile.model?.default ?? opts.cfg.model + const executorFactory: NonNullable = (spec, ctx) => { + let artifact: ExecutorResult | undefined + const executionId = ctx.node?.nodeId ?? `authored-router-${profile.name}` + const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(executionId) + const inner: Executor = attestRuntimeOwnedExecutor( + { + runtime: 'router', + async execute(_t, signal) { + const res = await routerChatWithUsage( + { ...opts.cfg, model }, + [ + { role: 'system', content: profile.prompt.systemPrompt }, + { role: 'user', content: opts.taskPrompt }, + ], + { temperature: opts.temperature ?? 0.4, ...(signal ? { signal } : {}) }, + ) + artifact = { + outRef: contentAddress(res.content), + out: res.content, + spent: { + iterations: 1, + tokens: res.usage ?? { input: 0, output: 0 }, + usd: res.costUsd ?? 0, + ms: 0, + }, + } + return artifact + }, + teardown: () => Promise.resolve({ destroyed: true }), + resultArtifact: () => { + if (!artifact) throw new Error('authoredWorker: resultArtifact read before execute') + return artifact + }, + }, + { + effectiveProfile: spec.profile, + backend: 'router', + model: { status: 'known', id: model }, + execution: { kind: 'request', id: executionId }, + materializer: 'authored-router-prompt', + plan: { + kind: 'authored-router-completion', + model, + temperature: opts.temperature ?? 0.4, + taskPrompt: opts.taskPrompt, }, - } - return artifact - }, - teardown: () => Promise.resolve({ destroyed: true }), - resultArtifact: () => { - if (!artifact) throw new Error('authoredWorker: resultArtifact read before execute') - return artifact - }, + }, + { + attemptId, + binding: { endpoint: opts.cfg.routerBaseUrl, executionId, model }, + descriptor: { kind: 'router-request', transport: 'http', backend: 'router' }, + }, + ) + return gateOnDeliverable(inner, opts.deliverable) } - const gated = gateOnDeliverable(inner, opts.deliverable) const spec: AgentSpec = { - profile: { name: profile.name } as AgentProfile, + profile, harness: null, - executor: gated, + executorFactory, } return { name: profile.name, act: async () => '', executorSpec: spec } as Agent< unknown, diff --git a/src/runtime/supervise/completion-gate.ts b/src/runtime/supervise/completion-gate.ts index 35a7e408..c785b2f1 100644 --- a/src/runtime/supervise/completion-gate.ts +++ b/src/runtime/supervise/completion-gate.ts @@ -21,6 +21,7 @@ * @experimental */ +import { inheritRuntimeOwnedExecutorAttestation } from './materialization' import type { DefaultVerdict, Executor, ExecutorResult, UsageEvent } from './types' /** @@ -58,7 +59,7 @@ export function gateOnDeliverable( return { valid: delivered, score: baseScore ?? (delivered ? 1 : 0) } } - return { + const wrapped: Executor = { runtime: inner.runtime, ...(inner.budgetExempt !== undefined ? { budgetExempt: inner.budgetExempt } : {}), ...(inner.deliver ? { deliver: (m: unknown) => inner.deliver?.(m) } : {}), @@ -97,6 +98,7 @@ export function gateOnDeliverable( return { ...art, verdict: gated ?? art.verdict } }, } + return inheritRuntimeOwnedExecutorAttestation(inner, wrapped) } function isAsyncIterable(v: unknown): v is AsyncIterable { diff --git a/src/runtime/supervise/coordination-log.ts b/src/runtime/supervise/coordination-log.ts index 61859ba3..867b7e00 100644 --- a/src/runtime/supervise/coordination-log.ts +++ b/src/runtime/supervise/coordination-log.ts @@ -1,14 +1,14 @@ /** - * Durable side-log for the coordination bus: questions and analyst findings, the two UP-leg - * message kinds the spawn journal does NOT record (it owns spawns/settlements/waits/spend). A - * durable run (`supervise({ runDir })`) appends them as they publish and replays them on resume, - * so a restarted coordinator still sees the questions its workers raised and the findings its - * analysts produced — instead of losing both with the process. + * Durable side-log for coordination evidence the spawn journal does not own: questions, analyst + * findings, answer decisions, authorized continuation receipts, delivery-attempt markers, and + * delivery outcomes. A durable run + * (`supervise({ runDir })`) appends them as they publish and loads them on resume, so a restarted + * coordinator retains the exact evidence produced by prior processes. * - * Answer down-events are logged too, ONLY to fold status on load: a question answered before the - * crash reloads as `answered`, not as a re-blocking `open`. Settled events are skipped (the spawn - * journal is their ledger); steer events are skipped (a delivered steer to a dead worker has no - * meaning to a new process). + * Answer down-events also fold status on load: a question answered before the crash reloads as + * `answered`, not as a re-blocking `open`. Settled events are skipped (the spawn journal is their + * ledger). A receipt followed by an attempt but no outcome proves the process died in the delivery + * window; that outcome remains unknown and no prior instruction is auto-delivered. * * JSONL, one fsynced record per event, keyed by `runId` — several runs may share one log file * exactly as they share one spawn-journal file. @@ -16,93 +16,201 @@ * @experimental */ +import { + parseCommittedJsonLines, + prepareJsonlAppend, + writeAllBytes, +} from '../../durable/jsonl-file' import type { AnalystFindingEvent, + ContinuationInstruction, CoordinationEvent, QuestionRecord, } from '../../mcp/tools/coordination' +import type { BusRecord } from './event-bus' -/** What a prior process's coordination log replays into a resumed driver. */ +/** Stable identity of the supervisor that owns one coordination stream. High-level supervision + * derives it from the exact root/child execution identity plus its parent assignment. */ +export type CoordinationOwnerId = string + +/** Durable delivery evidence retained in commit order. An attempt without a later event carrying + * the same `receiptId` has an unknown outcome after a crash and is never replayed. */ +export type CoordinationDeliveryEvidence = Extract< + CoordinationEvent, + { readonly type: 'delivery-attempt' | 'steer' | 'answer' } +> + +/** Coordination evidence loaded from prior processes of one durable supervised run. */ export interface PriorCoordination { + /** The owner filter used for this replay. Omitted only for the compatibility all-owner read. */ + readonly ownerId?: CoordinationOwnerId /** Every question the prior process raised, with answer-status folded in, raise order. */ readonly questions: ReadonlyArray /** Every analyst finding the prior process published, publish order. */ readonly findings: ReadonlyArray + /** Every authorized continuation, in commit order. These are evidence, never replayed to a new + * worker automatically. */ + readonly continuations: ReadonlyArray + /** Delivery intent and result records in commit order, linked to receipts by `receiptId`. */ + readonly deliveryEvidence: ReadonlyArray + /** Exact source-bus stamps in durable append order. Bus `seq` restarts with each process; append + * order remains the cross-process replay order. */ + readonly records: ReadonlyArray> } /** The durable coordination side-log seam. `append` records one bus event (kinds it does not * persist are ignored); `load` replays a run's prior records folded into `PriorCoordination`. */ export interface CoordinationLog { - append(runId: string, event: CoordinationEvent, at: string): Promise - load(runId: string): Promise + append( + runId: string, + record: BusRecord, + ownerId?: CoordinationOwnerId, + ): Promise + load(runId: string, ownerId?: CoordinationOwnerId): Promise } type CoordinationLogRecord = { readonly runId: string + readonly ownerId?: CoordinationOwnerId + readonly seq: number + readonly at: number + readonly priority: number + readonly event: CoordinationEvent +} + +type LegacyCoordinationLogRecord = { + readonly runId: string + readonly ownerId?: CoordinationOwnerId readonly at: string readonly event: CoordinationEvent } -/** Should this bus event be persisted? Questions and findings ARE the prior context a resumed - * driver needs; answers fold their status; everything else has a better ledger or none. */ +/** Persist prior context plus exact continuation authorization, attempt, and result evidence. + * Settlements have their own journal. */ function persisted(event: CoordinationEvent): boolean { - return event.type === 'question' || event.type === 'finding' || event.type === 'answer' + return event.type !== 'settled' } /** FS-backed `CoordinationLog`: append-only JSONL, fsynced per record. */ export class FileCoordinationLog implements CoordinationLog { + private appendTail: Promise = Promise.resolve() + constructor(private readonly path: string) {} - async append(runId: string, event: CoordinationEvent, at: string): Promise { - if (!persisted(event)) return + async append( + runId: string, + record: BusRecord, + ownerId?: CoordinationOwnerId, + ): Promise { + if (!persisted(record.event)) return + const append = this.appendTail.then(() => this.appendRecord(runId, record, ownerId)) + this.appendTail = append.catch(() => undefined) + return append + } + + private async appendRecord( + runId: string, + busRecord: BusRecord, + ownerId?: CoordinationOwnerId, + ): Promise { const fs = await import('node:fs/promises') const path = await import('node:path') await fs.mkdir(path.dirname(this.path), { recursive: true }) - const record: CoordinationLogRecord = { runId, at, event } + const record: CoordinationLogRecord = { + runId, + ...(ownerId !== undefined ? { ownerId } : {}), + ...busRecord, + } + const needsSeparator = await prepareJsonlAppend(this.path) const fh = await fs.open(this.path, 'a') try { - await fh.write(`${JSON.stringify(record)}\n`) + await writeAllBytes(fh, `${needsSeparator ? '\n' : ''}${JSON.stringify(record)}\n`) await fh.sync() } finally { await fh.close() } } - async load(runId: string): Promise { + async load(runId: string, ownerId?: CoordinationOwnerId): Promise { const fs = await import('node:fs/promises') let text: string try { text = await fs.readFile(this.path, 'utf8') } catch (err) { - if (isNoEntError(err)) return { questions: [], findings: [] } + if (isNoEntError(err)) return emptyPriorCoordination(ownerId) throw err } const byId = new Map() const findings: AnalystFindingEvent[] = [] - for (const line of text.split('\n')) { - if (line.length === 0) continue - const record = JSON.parse(line) as CoordinationLogRecord - if (record.runId !== runId) continue + const continuations: ContinuationInstruction[] = [] + const deliveryEvidence: CoordinationDeliveryEvidence[] = [] + const records: BusRecord[] = [] + let legacySeq = 0 + for (const stored of parseCommittedJsonLines< + CoordinationLogRecord | LegacyCoordinationLogRecord + >(text, this.path)) { + if (stored.runId !== runId) continue + // Omitting ownerId preserves the historical all-run read for direct consumers. Runtime always + // supplies one, so root and nested supervisors never receive one another's evidence. + if (ownerId !== undefined && stored.ownerId !== ownerId) continue + const record: BusRecord = + 'seq' in stored + ? { + seq: stored.seq, + at: stored.at, + priority: stored.priority, + event: stored.event, + } + : { + seq: legacySeq++, + at: Date.parse(stored.at), + priority: 0, + event: stored.event, + } + records.push(record) const ev = record.event + if (ev.type === 'delivery-attempt' || ev.type === 'steer' || ev.type === 'answer') { + deliveryEvidence.push(ev) + } if (ev.type === 'question') { byId.set(ev.question.id, ev.question) } else if (ev.type === 'finding') { findings.push(ev.finding) } else if (ev.type === 'answer') { - // Fold the answer into the question it decided, so an answered blocking question does - // not reload as open and re-block the resumed run's stop. `by` is not on the wire — - // the honest attribution is the prior run itself. + // Only accepted delivery resolves a blocking question. Authorization and an attempted + // refusal remain evidence, but cannot turn "worker never received the answer" into answered + // state on replay. `by` is not on the wire; prior-run is the honest attribution. const prior = byId.get(ev.questionId) - if (prior) { + if (prior && ev.down.delivered) { byId.set(ev.questionId, { ...prior, status: 'answered', decision: { kind: 'answer', answer: ev.down.instruction, by: 'prior-run' }, }) } + } else if (ev.type === 'instruction') { + continuations.push(ev.instruction) } } - return { questions: [...byId.values()], findings } + return { + ...(ownerId !== undefined ? { ownerId } : {}), + questions: [...byId.values()], + findings, + continuations, + deliveryEvidence, + records, + } + } +} + +function emptyPriorCoordination(ownerId?: CoordinationOwnerId): PriorCoordination { + return { + ...(ownerId !== undefined ? { ownerId } : {}), + questions: [], + findings: [], + continuations: [], + deliveryEvidence: [], + records: [], } } diff --git a/src/runtime/supervise/materialization.ts b/src/runtime/supervise/materialization.ts new file mode 100644 index 00000000..508f6a3f --- /dev/null +++ b/src/runtime/supervise/materialization.ts @@ -0,0 +1,361 @@ +import { randomUUID } from 'node:crypto' +import { + agentProfileSchema, + canonicalCandidateDigest, + type Sha256Digest, +} from '@tangle-network/agent-interface' +import { ValidationError } from '../../errors' +import { detachedSnapshot } from './snapshot' +import type { + ExecutionBindingReceipt, + Executor, + ExecutorExecutionBinding, + ExecutorMaterialization, + ProfileMaterializationReceipt, + Runtime, + UnknownMaterializationReason, +} from './types' + +type RuntimeOwnedExecutorMaterialization = + | { + readonly kind: 'known' + readonly declaration: ExecutorMaterialization + readonly binding?: ExecutorExecutionBinding + } + | { readonly kind: 'deferred'; readonly runtime: Runtime } + +const runtimeOwnedExecutorMaterializations = new WeakMap< + object, + RuntimeOwnedExecutorMaterialization +>() +const runtimeOwnedScopeOwners = new WeakMap() + +interface KnownReceiptInput { + readonly authoredProfileDigest: Sha256Digest + readonly runtime: Runtime + readonly declaration: ExecutorMaterialization +} + +interface UnknownReceiptInput { + readonly authoredProfileDigest?: Sha256Digest + readonly runtime: Runtime + readonly reason: UnknownMaterializationReason +} + +const declarationKeys = new Set([ + 'effectiveProfile', + 'backend', + 'model', + 'execution', + 'materializer', + 'plan', + 'platformAttachments', +]) + +/** + * Bind a declaration to a runtime-owned executor without putting an attestable method on the + * public Executor interface. This module is not a package export: arbitrary caller executors stay + * unknown even if they add a lookalike `materialization()` property. + */ +export function attestRuntimeOwnedExecutor( + executor: Executor, + declaration: ExecutorMaterialization, + binding?: ExecutorExecutionBinding, +): Executor { + runtimeOwnedExecutorMaterializations.set( + executor as object, + Object.freeze({ + kind: 'known', + declaration: detachedSnapshot(declaration, 'runtime-owned executor materialization'), + ...(binding === undefined + ? {} + : { + binding: detachedSnapshot(binding, 'runtime-owned executor execution binding'), + }), + }), + ) + return executor +} + +/** Mark a runtime-owned orchestration executor whose exact leaf declaration is published through + * its private nested Scope before the first backend inference. Arbitrary executors cannot opt in. */ +export function attestRuntimeOwnedDeferredExecutor( + executor: Executor, + runtime: Runtime, +): Executor { + runtimeOwnedExecutorMaterializations.set( + executor as object, + Object.freeze({ kind: 'deferred', runtime }), + ) + return executor +} + +/** Preserve the runtime-owned attestation when a trusted wrapper changes result semantics only. */ +export function inheritRuntimeOwnedExecutorAttestation( + source: Executor, + wrapper: Executor, +): Executor { + const attestation = runtimeOwnedExecutorMaterializations.get(source as object) + if (attestation !== undefined) { + runtimeOwnedExecutorMaterializations.set(wrapper as object, attestation) + } + return wrapper +} + +/** Read a declaration only when Runtime itself branded this exact executor object. */ +export function runtimeOwnedExecutorMaterialization( + executor: Executor, +): ExecutorMaterialization | undefined { + const attestation = runtimeOwnedExecutorMaterializations.get(executor as object) + return attestation?.kind === 'known' ? attestation.declaration : undefined +} + +/** Read one attempt binding only from the private brand on this exact executor object. */ +export function runtimeOwnedExecutorExecutionBinding( + executor: Executor, +): ExecutorExecutionBinding | undefined { + const attestation = runtimeOwnedExecutorMaterializations.get(executor as object) + return attestation?.kind === 'known' ? attestation.binding : undefined +} + +/** Read the expected leaf runtime only for a privately branded deferred orchestration executor. */ +export function runtimeOwnedDeferredExecutorRuntime( + executor: Executor, +): Runtime | undefined { + const attestation = runtimeOwnedExecutorMaterializations.get(executor as object) + return attestation?.kind === 'deferred' ? attestation.runtime : undefined +} + +/** Propagate trust from a built-in driver adapter to the Agent and recursive executor it owns. */ +export function attestRuntimeOwnedScopeOwner(owner: T, runtime: Runtime): T { + runtimeOwnedScopeOwners.set(owner, runtime) + return owner +} + +/** Return a built-in scope owner's expected concrete leaf runtime; caller-owned functions/Agents + * deliberately have no entry even if they add lookalike fields. */ +export function runtimeOwnedScopeOwnerRuntime(owner: object): Runtime | undefined { + return runtimeOwnedScopeOwners.get(owner) +} + +/** Build one kernel-owned known receipt from a data-only executor declaration. */ +export function knownMaterializationReceipt( + input: KnownReceiptInput, +): ProfileMaterializationReceipt { + const declaration = detachedSnapshot(input.declaration, 'executor materialization') + assertExactDeclaration(declaration) + const effectiveProfile = agentProfileSchema.safeParse(declaration.effectiveProfile) + if (!effectiveProfile.success) { + throw new ValidationError('executor materialization: effectiveProfile must be an AgentProfile') + } + assertNonEmpty(declaration.backend, 'backend') + assertNonEmpty(declaration.materializer, 'materializer') + assertNonEmpty(declaration.execution?.kind, 'execution.kind') + assertNonEmpty(declaration.execution?.id, 'execution.id') + assertModel(declaration.model) + + let effectiveProfileDigest: Sha256Digest + let materializationPlanDigest: Sha256Digest + let platformAttachmentsDigest: Sha256Digest | undefined + try { + // AgentProfile parsing represents omitted optional fields as `undefined`, while the backend's + // JSON request omits them. Commit the exact JSON document that crosses that boundary. + effectiveProfileDigest = canonicalCandidateDigest(jsonWireSnapshot(effectiveProfile.data)) + materializationPlanDigest = canonicalCandidateDigest(jsonWireSnapshot(declaration.plan)) + platformAttachmentsDigest = + declaration.platformAttachments === undefined + ? undefined + : canonicalCandidateDigest(jsonWireSnapshot(declaration.platformAttachments)) + } catch (error) { + throw new ValidationError( + 'executor materialization: profile, plan, and attachments must be finite RFC 8785 JSON', + { cause: error }, + ) + } + + return Object.freeze({ + status: 'known' as const, + authoredProfileDigest: input.authoredProfileDigest, + effectiveProfileDigest, + materializationPlanDigest, + ...(platformAttachmentsDigest === undefined ? {} : { platformAttachmentsDigest }), + runtime: input.runtime, + backend: declaration.backend, + model: declaration.model, + execution: declaration.execution, + materializer: declaration.materializer, + }) +} + +/** Build an explicit unknown receipt without fabricating any executor-owned identity. */ +export function unknownMaterializationReceipt( + input: UnknownReceiptInput, +): ProfileMaterializationReceipt { + return Object.freeze({ + status: 'unknown' as const, + ...(input.authoredProfileDigest === undefined + ? {} + : { authoredProfileDigest: input.authoredProfileDigest }), + runtime: input.runtime, + reason: input.reason, + }) +} + +/** Mint a process-unique attempt id before executor construction. */ +export function newExecutionAttemptId(nodeId: string): string { + return `${nodeId}:attempt:${randomUUID()}` +} + +/** Build the immutable safe receipt for a full, transient execution binding. */ +export function knownExecutionBindingReceipt( + materialization: ProfileMaterializationReceipt, + bindingInput: ExecutorExecutionBinding, +): ExecutionBindingReceipt { + const binding = detachedSnapshot(bindingInput, 'executor execution binding') + assertNonEmpty(binding.attemptId, 'binding.attemptId') + assertSafeDescriptor(binding.descriptor) + let materializationReceiptDigest: Sha256Digest + let bindingDigest: Sha256Digest + try { + materializationReceiptDigest = canonicalCandidateDigest(materialization) + // Bind the exact JSON document a backend receives. Optional AgentProfile fields are represented + // as `undefined` in memory but omitted by JSON transport; normalize that one distinction while + // rejecting every non-finite/non-data value instead of silently coercing it. + bindingDigest = canonicalCandidateDigest(jsonWireSnapshot(binding.binding)) + } catch (error) { + throw new ValidationError('executor binding must be finite RFC 8785 JSON', { cause: error }) + } + return Object.freeze({ + status: 'known' as const, + attemptId: binding.attemptId, + materializationReceiptDigest, + bindingDigest, + descriptor: binding.descriptor, + }) +} + +/** Preserve an attempt boundary even when its caller-owned transport cannot be attested. */ +export function unknownExecutionBindingReceipt( + materialization: ProfileMaterializationReceipt, + attemptId: string, + reason: UnknownMaterializationReason, +): ExecutionBindingReceipt { + assertNonEmpty(attemptId, 'binding.attemptId') + return Object.freeze({ + status: 'unknown' as const, + attemptId, + materializationReceiptDigest: canonicalCandidateDigest(materialization), + reason, + }) +} + +/** Derive an authored-profile digest without turning non-canonical input into a fake identity. */ +export function authoredProfileDigest(profile: unknown): Sha256Digest | undefined { + try { + return canonicalCandidateDigest(profile) + } catch { + return undefined + } +} + +function assertExactDeclaration(value: ExecutorMaterialization): void { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new ValidationError('executor materialization: declaration must be an object') + } + const unknown = Object.keys(value).filter((key) => !declarationKeys.has(key)) + if (unknown.length > 0) { + throw new ValidationError( + `executor materialization: unknown declaration fields: ${unknown.sort().join(', ')}`, + ) + } + if (!Object.hasOwn(value, 'plan')) { + throw new ValidationError('executor materialization: plan is required') + } +} + +function assertModel(model: ExecutorMaterialization['model']): void { + if (typeof model !== 'object' || model === null || Array.isArray(model)) { + throw new ValidationError('executor materialization: model must be a known/unknown identity') + } + const keys = Object.keys(model).sort() + if (model.status === 'known') { + if (keys.join(',') !== 'id,status') { + throw new ValidationError('executor materialization: known model accepts only status and id') + } + assertNonEmpty(model.id, 'model.id') + return + } + if (model.status === 'unknown') { + if (keys.join(',') !== 'reason,status') { + throw new ValidationError( + 'executor materialization: unknown model accepts only status and reason', + ) + } + assertNonEmpty(model.reason, 'model.reason') + return + } + throw new ValidationError('executor materialization: model.status must be known or unknown') +} + +function assertNonEmpty(value: unknown, field: string): asserts value is string { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new ValidationError(`executor materialization: ${field} must be a non-empty string`) + } +} + +function assertSafeDescriptor(descriptor: ExecutorExecutionBinding['descriptor']): void { + if (typeof descriptor !== 'object' || descriptor === null || Array.isArray(descriptor)) { + throw new ValidationError('executor binding: descriptor must be an object') + } + for (const [key, value] of Object.entries(descriptor)) { + if (/(url|uri|token|key|secret|bearer|credential|authorization)/i.test(key)) { + throw new ValidationError(`executor binding: unsafe descriptor key ${JSON.stringify(key)}`) + } + if ( + value !== null && + typeof value !== 'string' && + typeof value !== 'number' && + typeof value !== 'boolean' + ) { + throw new ValidationError( + `executor binding: descriptor ${JSON.stringify(key)} must be scalar`, + ) + } + if (typeof value === 'string' && (value.includes('://') || value.includes('@'))) { + throw new ValidationError( + `executor binding: descriptor ${JSON.stringify(key)} contains transport or credential text`, + ) + } + } +} + +function jsonWireSnapshot(value: unknown, path = '$'): unknown { + if (value === null || typeof value === 'string' || typeof value === 'boolean') return value + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new ValidationError(`executor binding: ${path} must be finite`) + } + return value + } + if (Array.isArray(value)) { + return value.map((item, index) => { + if (item === undefined) { + throw new ValidationError(`executor binding: ${path}[${index}] cannot be undefined`) + } + return jsonWireSnapshot(item, `${path}[${index}]`) + }) + } + if (typeof value !== 'object' || value === null) { + throw new ValidationError(`executor binding: ${path} must be JSON data`) + } + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) { + throw new ValidationError(`executor binding: ${path} must be a plain object`) + } + const out: Record = {} + for (const [key, item] of Object.entries(value as Record)) { + if (item === undefined) continue + out[key] = jsonWireSnapshot(item, `${path}.${key}`) + } + return out +} diff --git a/src/runtime/supervise/pi-executor.ts b/src/runtime/supervise/pi-executor.ts index a8bcc3a1..21f11a4b 100644 --- a/src/runtime/supervise/pi-executor.ts +++ b/src/runtime/supervise/pi-executor.ts @@ -24,16 +24,18 @@ * are read structurally off JSON lines, so a pi that adds commands stays compatible and a pi that * is not installed simply fails loud at spawn instead of at import. * - * Usage accounting: pi reports token usage on its assistant messages when the provider supplies - * it. Nothing is fabricated — a turn whose usage pi does not report contributes an `iteration` - * event and zero tokens, exactly like the other honest executors. + * Usage accounting: pi repeats the completed assistant message at `message_end` and `turn_end`. + * Only `turn_end` is authoritative for token and cost accounting. Nothing is fabricated — a + * turn whose usage pi does not report contributes an `iteration` event and marks telemetry unknown. * * @experimental */ import { type ChildProcess, spawn } from 'node:child_process' +import { randomUUID } from 'node:crypto' import { ValidationError } from '../../errors' import { createInbox, type Inbox, type InboxMessage } from './inbox' +import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' import { type ActivityLog, createActivityLog, type ExecutorProgress } from './progress' import { createPushTraceSource, type ToolStepInput, type TraceSource } from './trace-source' import type { @@ -80,12 +82,20 @@ interface PiEvent { /** Build the `Executor` for one pi worker. Registered as runtime `'pi'`. */ export const piExecutor: ExecutorFactory = (spec, ctx) => { - const seam = readPiSeam(ctx) + const configured = readPiSeam(ctx) + const seam: PiSeam = { + ...configured, + // The backend model is a fallback for profiles that do not select one. AgentProfile is the + // experiment-owned knob, so an ambient/default seam must never override the authored arm. + ...(spec.profile.model?.default ? { model: spec.profile.model.default } : {}), + } const inbox = createInbox() const activity = createActivityLog(seam.activityWindow ?? 12) const trace = createPushTraceSource({ runId: `pi-${spec.profile.name ?? 'worker'}-${Date.now()}`, }) + const executionId = ctx.node?.nodeId ?? `pi-run-${randomUUID()}` + const attemptId = ctx.node?.attemptId ?? newExecutionAttemptId(executionId) const controller = new AbortController() const cascade = () => controller.abort() @@ -100,50 +110,79 @@ export const piExecutor: ExecutorFactory = (spec, ctx) => { artifact: undefined as ExecutorResult | undefined, } - return { - runtime: PI_RUNTIME, - // pi owns the queue; `deliver` only routes through its state-safe `prompt` command. Its - // streaming behavior chooses steer versus follow-up atomically in pi, rather than trusting - // this adapter's delayed view of whether the current run has already ended. - deliver: (m) => inbox.deliver(m), - progress: (): ExecutorProgress => ({ - turns: state.turns, - pendingMessages: inbox.pending(), - recentActivity: activity.read(), - note: state.note, - }), - traceSource: (): TraceSource => trace.source, - execute(task, signal): AsyncIterable { - return streamPiSession({ - task, - signal, - controller, - seam, - spec, - inbox, - activity, - record: (step: ToolStepInput) => { - trace.record(step) - }, - state, - }) + return attestRuntimeOwnedExecutor( + { + runtime: PI_RUNTIME, + // pi owns the queue; `deliver` only routes through its state-safe `prompt` command. Its + // streaming behavior chooses steer versus follow-up atomically in pi, rather than trusting + // this adapter's delayed view of whether the current run has already ended. + deliver: (m) => inbox.deliver(m), + progress: (): ExecutorProgress => ({ + turns: state.turns, + pendingMessages: inbox.pending(), + recentActivity: activity.read(), + note: state.note, + }), + traceSource: (): TraceSource => trace.source, + execute(task, signal): AsyncIterable { + return streamPiSession({ + task, + signal, + controller, + seam, + spec, + inbox, + activity, + record: (step: ToolStepInput) => { + trace.record(step) + }, + state, + }) + }, + async teardown(grace): Promise<{ destroyed: boolean }> { + controller.abort() + const proc = state.proc + if (!proc || proc.exitCode !== null || proc.killed) return { destroyed: true } + // Ask pi to stop cleanly first — an `abort` lets it finalize its session JSONL, which is + // the whole reason to wrap pi rather than kill it. + sendCommand(proc, { type: 'abort' }) + return killPi(proc, grace) + }, + resultArtifact() { + if (!state.artifact) { + throw new ValidationError('piExecutor: resultArtifact() read before stream drained') + } + return state.artifact + }, }, - async teardown(grace): Promise<{ destroyed: boolean }> { - controller.abort() - const proc = state.proc - if (!proc || proc.exitCode !== null || proc.killed) return { destroyed: true } - // Ask pi to stop cleanly first — an `abort` lets it finalize its session JSONL, which is - // the whole reason to wrap pi rather than kill it. - sendCommand(proc, { type: 'abort' }) - return killPi(proc, grace) + { + effectiveProfile: spec.profile, + backend: 'pi', + model: seam.model + ? { status: 'known', id: seam.model } + : { status: 'unknown', reason: 'pi selected its configured default model' }, + execution: { kind: 'run', id: executionId }, + materializer: 'pi-rpc-agent-profile', + plan: { + kind: 'pi-rpc-session', + bin: seam.bin ?? 'pi', + args: seam.args ?? [], + cwd: seam.cwd ?? null, + model: seam.model ?? null, + turnTimeoutMs: seam.turnTimeoutMs ?? null, + }, }, - resultArtifact() { - if (!state.artifact) { - throw new ValidationError('piExecutor: resultArtifact() read before stream drained') - } - return state.artifact + { + attemptId, + binding: { + executionId, + bin: seam.bin ?? 'pi', + cwd: seam.cwd ?? null, + model: seam.model ?? null, + }, + descriptor: { kind: 'pi-rpc-run', transport: 'process', backend: 'pi' }, }, - } + ) } interface StreamPiArgs { @@ -151,7 +190,9 @@ interface StreamPiArgs { signal: AbortSignal controller: AbortController seam: PiSeam - spec: { profile: { name?: string; prompt?: { systemPrompt?: string } } } + spec: { + profile: { name?: string; prompt?: { systemPrompt?: string; instructions?: string[] } } + } inbox: Inbox activity: ActivityLog record: (step: ToolStepInput) => void @@ -174,6 +215,10 @@ async function* streamPiSession(args: StreamPiArgs): AsyncIterable { const { seam, inbox, activity, state } = args const started = Date.now() const tokens = { input: 0, output: 0 } + let tokensKnown = true + let usdKnown = true + let turnTokensKnown = false + let turnUsdKnown = false let usd = 0 const proc = spawnPi(seam) @@ -222,7 +267,12 @@ async function* streamPiSession(args: StreamPiArgs): AsyncIterable { args.controller.signal.addEventListener('abort', abortAll, { once: true }) } - const system = args.spec.profile.prompt?.systemPrompt + const system = [ + args.spec.profile.prompt?.systemPrompt, + ...(args.spec.profile.prompt?.instructions ?? []), + ] + .filter((line): line is string => typeof line === 'string' && line.trim().length > 0) + .join('\n') const opening = system ? `${system}\n\n${taskText(args.task)}` : taskText(args.task) const deadline = seam.turnTimeoutMs ? Date.now() + seam.turnTimeoutMs : undefined @@ -239,10 +289,21 @@ async function* streamPiSession(args: StreamPiArgs): AsyncIterable { // Drain what pi has emitted so far, projecting usage + activity. while (events.length > 0) { const ev = events.shift() as PiEvent + if (ev.type === 'turn_end') { + const telemetry = readUsage(ev.message) + if (telemetry?.tokensKnown) turnTokensKnown = true + if (telemetry?.usdKnown) turnUsdKnown = true + } for (const usage of projectPiEvent(ev, args, tokens)) { if (usage.kind === 'cost') usd += usage.usd yield usage } + if (ev.type === 'turn_end') { + if (!turnTokensKnown) tokensKnown = false + if (!turnUsdKnown) usdKnown = false + turnTokensKnown = false + turnUsdKnown = false + } } if (failure) throw failure @@ -282,7 +343,14 @@ async function* streamPiSession(args: StreamPiArgs): AsyncIterable { await killPi(proc, 2_000).catch(() => ({ destroyed: false })) } - const spent: Spend = { iterations: state.turns, tokens, usd, ms: Date.now() - started } + const spent: Spend = { + iterations: state.turns, + tokens, + ...(tokensKnown ? {} : { tokensKnown: false }), + usd, + ...(usdKnown ? {} : { usdKnown: false }), + ms: Date.now() - started, + } state.artifact = { outRef: `pi:${hash(state.lastText)}`, out: { content: state.lastText, turns: state.turns }, @@ -340,13 +408,15 @@ function projectPiEvent( return out } if (ev.type === 'turn_end' || ev.type === 'message_end') { - const usage = readUsage(ev.message) - if (usage && (usage.input || usage.output)) { - tokens.input += usage.input - tokens.output += usage.output - out.push({ kind: 'tokens', input: usage.input, output: usage.output }) + if (ev.type === 'turn_end') { + const usage = readUsage(ev.message) + if (usage && (usage.input || usage.output)) { + tokens.input += usage.input + tokens.output += usage.output + out.push({ kind: 'tokens', input: usage.input, output: usage.output }) + } + if (usage?.usd) out.push({ kind: 'cost', usd: usage.usd }) } - if (usage?.usd) out.push({ kind: 'cost', usd: usage.usd }) const text = readText(ev.message) if (text) args.state.lastText = text if (ev.type === 'turn_end') { @@ -360,20 +430,37 @@ function projectPiEvent( /** pi's assistant message carries provider usage when the provider reported it. Absent, nothing * is fabricated — the turn still counts as an iteration with zero tokens. */ -function readUsage(message: unknown): { input: number; output: number; usd: number } | undefined { +function readUsage(message: unknown): + | { + input: number + output: number + usd: number + tokensKnown: boolean + usdKnown: boolean + } + | undefined { if (!message || typeof message !== 'object') return undefined const usage = (message as { usage?: unknown }).usage if (!usage || typeof usage !== 'object') return undefined const u = usage as Record - const input = num(u.input) ?? num(u.inputTokens) ?? num(u.prompt_tokens) ?? 0 - const output = num(u.output) ?? num(u.outputTokens) ?? num(u.completion_tokens) ?? 0 + const inputValue = num(u.input) ?? num(u.inputTokens) ?? num(u.prompt_tokens) + const outputValue = num(u.output) ?? num(u.outputTokens) ?? num(u.completion_tokens) + const input = inputValue ?? 0 + const output = outputValue ?? 0 const costRaw = u.cost - const usd = + const usdValue = num(costRaw) ?? (costRaw && typeof costRaw === 'object' - ? (num((costRaw as Record).totalCost) ?? 0) - : 0) - return { input, output, usd } + ? (num((costRaw as Record).total) ?? + num((costRaw as Record).totalCost)) + : undefined) + return { + input, + output, + usd: usdValue ?? 0, + tokensKnown: inputValue !== undefined && outputValue !== undefined, + usdKnown: usdValue !== undefined, + } } function readText(message: unknown): string | undefined { diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 27f771b8..f8d7eb01 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -1170,7 +1170,6 @@ export const bridgeExecutor: ExecutorFactory = (spec, ctx) => { materializer: 'cli-bridge-agent-profile', plan: { kind: 'cli-bridge-session', - bridgeUrl: seam.bridgeUrl, cwd: seam.cwd ?? null, maxTurns, timeoutMs: seam.timeoutMs ?? null, @@ -2077,7 +2076,6 @@ function bridgeWorktreeExecutor( runId, sessionId, baseRef: seam.baseRef ?? 'HEAD', - bridgeUrl: bridge.bridgeUrl, model: model ?? null, testCmd: seam.testCmd ?? null, typecheckCmd: seam.typecheckCmd ?? null, diff --git a/src/runtime/supervise/worktree-cli-executor.ts b/src/runtime/supervise/worktree-cli-executor.ts index 88df588d..d885885f 100644 --- a/src/runtime/supervise/worktree-cli-executor.ts +++ b/src/runtime/supervise/worktree-cli-executor.ts @@ -33,7 +33,9 @@ import { type WorktreeHarnessResult, type WorktreeHarnessRun, type WorktreeProfileMaterializationReceipt, + worktreeProfileExecutionPlan, } from '../../mcp/worktree-harness' +import { attestRuntimeOwnedExecutor, newExecutionAttemptId } from './materialization' import type { Executor, ExecutorResult, Spend } from './types' export type { WorktreeCommandResult, WorktreeProfileMaterializationReceipt } @@ -55,8 +57,9 @@ export interface WorktreeCliExecutorOptions { profile: AgentProfile /** Local CLI for this leaf. This explicit choice overrides `profile.harness`. */ harness: LocalHarness - /** The per-task instruction handed to the harness (composed under the system prompt). */ - taskPrompt: string + /** Default instruction for direct `execute(undefined, signal)` calls. An execution-time task + * is authoritative. Omit when the caller always supplies the task to `execute`. */ + taskPrompt?: string /** Unique id for the worktree path + branch. Defaults to a fresh UUID. */ runId?: string /** Override the base ref the worktree is cut from (default `HEAD`). */ @@ -93,14 +96,17 @@ export interface WorktreeCliExecutorOptions { * likewise return `LocalHarnessResult.usage`. */ budgetExempt?: boolean + /** @internal Kernel-minted attempt identity threaded by the built-in registry. */ + executionAttemptId?: string } /** * Build a worktree-CLI leaf `Executor`. Per-spawn (a fresh worktree + abort + teardown each), so a * fanout of N profiles = N parallel worktrees that never clobber each other. * - * Fail-loud: an empty `repoRoot`/`harness`/`taskPrompt` throws at construction. `resultArtifact()` - * before `execute()` resolves throws. + * Fail-loud: an empty `repoRoot`/`harness` or an explicitly empty `taskPrompt` throws at + * construction. Calling `execute(undefined, signal)` without a configured prompt throws before a + * worktree is created. `resultArtifact()` before `execute()` resolves throws. * * @experimental */ @@ -113,7 +119,10 @@ export function createWorktreeCliExecutor( if (!options.harness) { throw new ValidationError('createWorktreeCliExecutor: harness required') } - if (typeof options.taskPrompt !== 'string' || options.taskPrompt.length === 0) { + if ( + options.taskPrompt !== undefined && + (typeof options.taskPrompt !== 'string' || options.taskPrompt.length === 0) + ) { throw new ValidationError('createWorktreeCliExecutor: taskPrompt required') } if (options.codexReproducible && options.harness !== 'codex') { @@ -131,84 +140,151 @@ export function createWorktreeCliExecutor( } const runId = options.runId ?? randomUUID() + const attemptId = options.executionAttemptId ?? newExecutionAttemptId(runId) const controller = new AbortController() const budgetExempt = options.budgetExempt ?? !options.codexReproducible let run: WorktreeHarnessRun | undefined let artifact: ExecutorResult | undefined - return { - runtime: 'cli', - budgetExempt, - async execute(_task, signal): Promise> { - const linked = linkSignals(signal, controller.signal) - const started = Date.now() + const profilePlan = worktreeProfileExecutionPlan(options.profile, options.harness) + return attestRuntimeOwnedExecutor( + { + runtime: 'cli', + budgetExempt, + async execute(task, signal): Promise> { + const linked = linkSignals(signal, controller.signal) + const started = Date.now() + const taskPrompt = executionTaskPrompt(task, options.taskPrompt) - run = await runWorktreeHarness({ - repoRoot: options.repoRoot, - profile: options.profile, - harness: options.harness, - taskPrompt: options.taskPrompt, - runId, - ...(options.baseRef ? { baseRef: options.baseRef } : {}), - ...(options.testCmd !== undefined ? { testCmd: options.testCmd } : {}), - ...(options.typecheckCmd !== undefined ? { typecheckCmd: options.typecheckCmd } : {}), - ...(options.harnessTimeoutMs !== undefined - ? { harnessTimeoutMs: options.harnessTimeoutMs } - : {}), - ...(options.codexReproducible ? { codexReproducible: true } : {}), - ...(options.codexReadDeniedPaths - ? { codexReadDeniedPaths: options.codexReadDeniedPaths } - : {}), - ...(options.checkTimeoutMs !== undefined ? { checkTimeoutMs: options.checkTimeoutMs } : {}), - ...(options.checkOutputCap !== undefined ? { checkOutputCap: options.checkOutputCap } : {}), - ...(linked ? { signal: linked } : {}), - ...(options.runGit ? { runGit: options.runGit } : {}), - ...(options.runHarness ? { runHarness: options.runHarness } : {}), - ...(options.runCommand ? { runCommand: options.runCommand } : {}), - }) + run = await runWorktreeHarness({ + repoRoot: options.repoRoot, + profile: options.profile, + harness: options.harness, + taskPrompt, + runId, + ...(options.baseRef ? { baseRef: options.baseRef } : {}), + ...(options.testCmd !== undefined ? { testCmd: options.testCmd } : {}), + ...(options.typecheckCmd !== undefined ? { typecheckCmd: options.typecheckCmd } : {}), + ...(options.harnessTimeoutMs !== undefined + ? { harnessTimeoutMs: options.harnessTimeoutMs } + : {}), + ...(options.codexReproducible ? { codexReproducible: true } : {}), + ...(options.codexReadDeniedPaths + ? { codexReadDeniedPaths: options.codexReadDeniedPaths } + : {}), + ...(options.checkTimeoutMs !== undefined + ? { checkTimeoutMs: options.checkTimeoutMs } + : {}), + ...(options.checkOutputCap !== undefined + ? { checkOutputCap: options.checkOutputCap } + : {}), + ...(linked ? { signal: linked } : {}), + ...(options.runGit ? { runGit: options.runGit } : {}), + ...(options.runHarness ? { runHarness: options.runHarness } : {}), + ...(options.runCommand ? { runCommand: options.runCommand } : {}), + }) - const usage = run.result.harness.usage - if (!budgetExempt && !usage) { - const completed = run - run = undefined - await completed.cleanup() - throw new ValidationError( - 'createWorktreeCliExecutor: metered harness run returned no token usage', - ) - } - const spent: Spend = { - iterations: 1, - tokens: usage - ? { input: usage.inputTokens, output: usage.outputTokens } - : { input: 0, output: 0 }, - usd: 0, - ...(usage ? { usdKnown: false } : {}), - ms: Date.now() - started, - } - artifact = { outRef: contentAddress(run.result), out: run.result, spent } - return artifact + const usage = run.result.harness.usage + if (!budgetExempt && !usage) { + const completed = run + run = undefined + await completed.cleanup() + throw new ValidationError( + 'createWorktreeCliExecutor: metered harness run returned no token usage', + ) + } + const spent: Spend = { + iterations: 1, + tokens: usage + ? { input: usage.inputTokens, output: usage.outputTokens } + : { input: 0, output: 0 }, + usd: 0, + ...(usage ? { usdKnown: false } : {}), + ms: Date.now() - started, + } + artifact = { outRef: contentAddress(run.result), out: run.result, spent } + return artifact + }, + async teardown(_grace): Promise<{ destroyed: boolean }> { + controller.abort() + // The loser of a fanout (or any settled run) is dirty — remove its worktree. A run that + // THREW already cleaned itself up in the core, so `run` stays undefined and this is a no-op. + if (run) { + const r = run + run = undefined + await r.cleanup() + } + return { destroyed: true } + }, + resultArtifact() { + if (!artifact) { + throw new ValidationError( + 'createWorktreeCliExecutor: resultArtifact() read before execute() resolved', + ) + } + return artifact + }, }, - async teardown(_grace): Promise<{ destroyed: boolean }> { - controller.abort() - // The loser of a fanout (or any settled run) is dirty — remove its worktree. A run that - // THREW already cleaned itself up in the core, so `run` stays undefined and this is a no-op. - if (run) { - const r = run - run = undefined - await r.cleanup() - } - return { destroyed: true } + { + effectiveProfile: options.profile, + backend: `cli-worktree:${options.harness}`, + model: options.profile.model?.default + ? { status: 'known', id: options.profile.model.default } + : { status: 'unknown', reason: `${options.harness} selected its configured default model` }, + execution: { kind: 'worktree-run', id: runId }, + materializer: 'agent-profile-worktree-plan', + plan: { + kind: 'worktree-cli', + profilePlan, + harness: options.harness, + baseRef: options.baseRef ?? 'HEAD', + harnessTimeoutMs: options.harnessTimeoutMs ?? null, + codexReproducible: options.codexReproducible === true, + codexReadDeniedPaths: options.codexReadDeniedPaths ?? [], + testCmd: options.testCmd ?? null, + typecheckCmd: options.typecheckCmd ?? null, + checkTimeoutMs: options.checkTimeoutMs ?? null, + checkOutputCap: options.checkOutputCap ?? 16_000, + }, }, - resultArtifact() { - if (!artifact) { - throw new ValidationError( - 'createWorktreeCliExecutor: resultArtifact() read before execute() resolved', - ) - } - return artifact + { + attemptId, + binding: { + repoRoot: options.repoRoot, + runId, + harness: options.harness, + model: options.profile.model?.default ?? null, + baseRef: options.baseRef ?? 'HEAD', + }, + descriptor: { + kind: 'worktree-cli-run', + transport: 'process', + backend: options.harness, + }, }, + ) +} + +/** A scoped execution task is authoritative. The configured prompt remains only as the + * unambiguous direct-call default for existing `execute(undefined, signal)` consumers. */ +function executionTaskPrompt(task: unknown, configuredPrompt: string | undefined): string { + if (task === undefined) { + if (configuredPrompt !== undefined) return configuredPrompt + throw new ValidationError( + 'createWorktreeCliExecutor: execute task required when taskPrompt is not configured', + ) + } + if (typeof task === 'string') return task + try { + const encoded = JSON.stringify(task) + if (encoded !== undefined) return encoded + } catch (error) { + throw new ValidationError('createWorktreeCliExecutor: execute task must be JSON-serializable', { + cause: error, + }) } + throw new ValidationError('createWorktreeCliExecutor: execute task must be JSON-serializable') } /** Link two abort signals into one that fires when either does. Returns `undefined` when neither diff --git a/tests/kernel/materialization-evidence.test.ts b/tests/kernel/materialization-evidence.test.ts new file mode 100644 index 00000000..125a7458 --- /dev/null +++ b/tests/kernel/materialization-evidence.test.ts @@ -0,0 +1,268 @@ +import { canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { + InMemoryResultBlobStore, + InMemorySpawnJournal, + materializeTreeView, +} from '../../src/durable/spawn-journal' +import { driverChild, withDriverExecutor } from '../../src/runtime/supervise/driver-executor' +import { + attestRuntimeOwnedExecutor, + attestRuntimeOwnedScopeOwner, + knownExecutionBindingReceipt, + knownMaterializationReceipt, + runtimeOwnedExecutorExecutionBinding, + runtimeOwnedExecutorMaterialization, +} from '../../src/runtime/supervise/materialization' +import { bridgeExecutor, createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { createSupervisor } from '../../src/runtime/supervise/supervisor' +import type { + Agent, + AgentSpec, + Executor, + ExecutorFactory, + ExecutorResult, + Scope, + SpawnEvent, +} from '../../src/runtime/supervise/types' + +const budget = { maxIterations: 4, maxTokens: 1_000 } +const spent = { iterations: 1, tokens: { input: 2, output: 3 }, usd: 0, ms: 1 } + +function leafAgent(name: string, factory: ExecutorFactory): Agent { + const executorSpec: AgentSpec = { + profile: { name, model: { default: 'test/model' } }, + harness: null, + executorFactory: factory, + } + return { name, executorSpec, act: async () => undefined } as Agent & { + executorSpec: AgentSpec + } +} + +function successfulExecutor(runtime = 'test-runtime'): Executor { + const artifact: ExecutorResult = { outRef: 'ignored', out: { ok: true }, spent } + return { + runtime, + execute: async () => artifact, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => artifact, + } +} + +async function runOneChild( + child: Agent, + journal: InMemorySpawnJournal, + runId: string, + executors = createExecutorRegistry(), +) { + const root: Agent = { + name: 'root', + async act(_task, scope) { + expect('recordMaterialization' in scope).toBe(false) + expect(() => + ( + scope as Scope & { recordMaterialization: (value: unknown) => void } + ).recordMaterialization({ forged: true }), + ).toThrow(TypeError) + const spawned = scope.spawn(child, 'work', { label: 'child', budget }) + if (!spawned.ok) throw new Error(spawned.reason) + return scope.next() + }, + } + return createSupervisor().run(root, 'root task', { + budget: { maxIterations: 20, maxTokens: 20_000 }, + runId, + journal, + blobs: new InMemoryResultBlobStore(), + executors, + }) +} + +function childEvents(events: SpawnEvent[] | undefined): SpawnEvent[] { + return (events ?? []).filter((event) => event.id.endsWith(':s0')) +} + +describe('kernel-owned materialization evidence', () => { + it('ignores a caller executor that lies through lookalike report methods', async () => { + const journal = new InMemorySpawnJournal() + const liar = leafAgent('liar', (_spec, _ctx) => + Object.assign(successfulExecutor(), { + materialization: () => ({ status: 'known', backend: 'forged' }), + executionBinding: () => ({ bindingDigest: canonicalCandidateDigest({ forged: true }) }), + }), + ) + + await runOneChild(liar, journal, 'lying-executor') + const events = childEvents(await journal.loadTree('lying-executor')) + expect(events.find((event) => event.kind === 'materialized')).toMatchObject({ + receipt: { status: 'unknown', reason: 'executor-did-not-report' }, + }) + expect(events.find((event) => event.kind === 'execution-bound')).toMatchObject({ + binding: { status: 'unknown', reason: 'executor-did-not-report' }, + }) + }) + + it('records invalid trusted evidence and refuses execution', async () => { + const journal = new InMemorySpawnJournal() + let executeCalls = 0 + const invalid = leafAgent('invalid', (spec, ctx) => { + const executor = successfulExecutor() + const originalExecute = executor.execute.bind(executor) + executor.execute = (task, signal) => { + executeCalls += 1 + return originalExecute(task, signal) + } + return attestRuntimeOwnedExecutor( + executor, + { + effectiveProfile: spec.profile, + backend: '', + model: { status: 'known', id: 'test/model' }, + execution: { kind: 'run', id: ctx.node?.nodeId ?? 'direct' }, + materializer: 'test', + plan: { kind: 'test' }, + }, + { + attemptId: ctx.node?.attemptId ?? 'direct-attempt', + binding: { endpoint: 'https://example.test' }, + descriptor: { kind: 'test', transport: 'http' }, + }, + ) + }) + + await runOneChild(invalid, journal, 'invalid-evidence') + expect(executeCalls).toBe(0) + expect(childEvents(await journal.loadTree('invalid-evidence'))).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'materialized', + receipt: expect.objectContaining({ + status: 'unknown', + reason: 'invalid-executor-report', + }), + }), + expect.objectContaining({ + kind: 'execution-bound', + binding: expect.objectContaining({ + status: 'unknown', + reason: 'invalid-executor-report', + }), + }), + ]), + ) + }) + + it('projects the same trusted receipt and binding from journal replay', async () => { + const journal = new InMemorySpawnJournal() + const trusted = leafAgent('trusted', (spec, ctx) => { + const executionId = ctx.node?.nodeId ?? 'direct' + return attestRuntimeOwnedExecutor( + successfulExecutor('router'), + { + effectiveProfile: spec.profile, + backend: 'router', + model: { status: 'known', id: 'test/model' }, + execution: { kind: 'request', id: executionId }, + materializer: 'test-router', + plan: { kind: 'completion', model: 'test/model' }, + }, + { + attemptId: ctx.node?.attemptId ?? 'direct-attempt', + binding: { endpoint: 'https://router.example.test', executionId }, + descriptor: { kind: 'router-request', transport: 'http' }, + }, + ) + }) + + const result = await runOneChild(trusted, journal, 'known-evidence') + const events = await journal.loadTree('known-evidence') + const replayed = materializeTreeView(events ?? []) + const live = result.tree.nodes.find((node) => node.id.endsWith(':s0')) + const replay = replayed.nodes.find((node) => node.id.endsWith(':s0')) + expect(live?.materialization).toMatchObject({ status: 'known', backend: 'router' }) + expect(live?.executionBindings).toHaveLength(1) + expect(replay?.materialization).toEqual(live?.materialization) + expect(replay?.executionBindings).toEqual(live?.executionBindings) + }) + + it('types a trusted deferred manager down when it never publishes before completing', async () => { + const journal = new InMemorySpawnJournal() + const silentManager = attestRuntimeOwnedScopeOwner>( + { name: 'silent-manager', act: async () => ({ shouldNotWin: true }) }, + 'cli', + ) + const child = driverChild( + { name: 'manager', metadata: { role: 'driver' } }, + silentManager, + journal, + ) + const executors = withDriverExecutor(createExecutorRegistry()) + + await runOneChild(child, journal, 'missing-deferred', executors) + expect(childEvents(await journal.loadTree('missing-deferred'))).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'materialized', + receipt: expect.objectContaining({ + status: 'unknown', + reason: 'root-agent-did-not-report', + }), + }), + expect.objectContaining({ + kind: 'settled', + status: 'down', + }), + ]), + ) + }) + + it('keeps the built-in bridge profile identity stable while endpoints change per attempt', () => { + const profile = { name: 'manager', model: { default: 'test/model' } } + const executorFor = (bridgeUrl: string, attemptId: string) => + bridgeExecutor( + { profile, harness: null }, + { + signal: new AbortController().signal, + node: { + rootId: 'root', + parentId: 'root', + nodeId: 'root:s0', + attemptId, + }, + seams: { + bridge: { + bridgeUrl, + bridgeBearer: 'never-journal-this', + model: 'test/model', + sessionId: 'stable-session', + }, + }, + }, + ) + const receiptFor = (bridgeUrl: string, attemptId: string) => { + const executor = executorFor(bridgeUrl, attemptId) + const declaration = runtimeOwnedExecutorMaterialization(executor) + const binding = runtimeOwnedExecutorExecutionBinding(executor) + expect(declaration).toBeDefined() + expect(binding).toBeDefined() + const materialization = knownMaterializationReceipt({ + authoredProfileDigest: canonicalCandidateDigest(profile), + runtime: 'cli', + declaration: declaration!, + }) + return { + materialization, + binding: knownExecutionBindingReceipt(materialization, binding!), + } + } + const first = receiptFor('http://127.0.0.1:31001', 'attempt-1') + const second = receiptFor('http://127.0.0.1:31002', 'attempt-2') + + expect(first.materialization).toEqual(second.materialization) + expect(first.binding.materializationReceiptDigest).toBe( + second.binding.materializationReceiptDigest, + ) + expect(first.binding.bindingDigest).not.toBe(second.binding.bindingDigest) + }) +}) diff --git a/tests/runtime/spawn-journal-replay-identity.test.ts b/tests/runtime/spawn-journal-replay-identity.test.ts new file mode 100644 index 00000000..5438be39 --- /dev/null +++ b/tests/runtime/spawn-journal-replay-identity.test.ts @@ -0,0 +1,173 @@ +import { canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { + contentAddress, + InMemoryResultBlobStore, + InMemorySpawnJournal, + materializeTreeView, + replaySpawnTree, +} from '../../src/durable/spawn-journal' +import type { NodeExecutionIdentity, SpawnEvent } from '../../src/runtime/supervise/types' + +const spent = { + iterations: 1, + tokens: { input: 2, output: 3 }, + usd: 0.01, + ms: 4, +} + +describe('spawn journal replay identity', () => { + it('keeps a profile materialization receipt informational when rebuilding node status', () => { + const profileDigest = canonicalCandidateDigest({ profile: 'scientist' }) + const view = materializeTreeView([ + { + kind: 'spawned', + id: 'receipt-only', + label: 'root', + budget: { maxIterations: 1, maxTokens: 100 }, + runtime: 'cli', + seq: 0, + at: new Date(0).toISOString(), + }, + { + kind: 'materialized', + id: 'receipt-only', + receipt: { + status: 'known', + authoredProfileDigest: profileDigest, + effectiveProfileDigest: profileDigest, + materializationPlanDigest: canonicalCandidateDigest({ kind: 'test-plan' }), + runtime: 'cli', + backend: 'bridge', + model: { status: 'known', id: 'test/model' }, + execution: { kind: 'session', id: 'test-session' }, + materializer: 'test', + }, + seq: 0, + at: new Date(0).toISOString(), + }, + { + kind: 'execution-bound', + id: 'receipt-only', + binding: { + status: 'known', + attemptId: 'attempt-1', + materializationReceiptDigest: canonicalCandidateDigest({ receipt: 'test' }), + bindingDigest: canonicalCandidateDigest({ endpoint: 'hidden' }), + descriptor: { kind: 'bridge-session', transport: 'http' }, + }, + seq: 0, + at: new Date(0).toISOString(), + }, + ]) + + expect(view.nodes).toEqual([expect.objectContaining({ id: 'receipt-only', status: 'pending' })]) + }) + + it('copies every spawned identity field onto done, down, and cancelled terminal handles', async () => { + const journal = new InMemorySpawnJournal() + const blobs = new InMemoryResultBlobStore() + const root = 'identity-replay' + const at = new Date(0).toISOString() + const correlation = { pursuitId: 'pursuit-1', experimentId: 'experiment-1' } + const identity: NodeExecutionIdentity = { + profileDigest: canonicalCandidateDigest({ profile: 'scientist' }), + taskDigest: canonicalCandidateDigest({ task: 'test the mechanism' }), + candidateDigest: canonicalCandidateDigest({ candidate: 'candidate-1' }), + correlation, + } + const expectedIdentity = { + profileDigest: identity.profileDigest, + taskDigest: identity.taskDigest, + candidateDigest: identity.candidateDigest, + correlation: { ...correlation }, + } + + await journal.beginTree(root, at) + const spawns: SpawnEvent[] = [ + { + kind: 'spawned', + id: `${root}:done`, + parent: root, + label: 'done', + key: 'done-key', + budget: { maxIterations: 1, maxTokens: 100 }, + runtime: 'router', + identity, + seq: 0, + at, + }, + { + kind: 'spawned', + id: `${root}:down`, + parent: root, + label: 'down', + key: 'down-key', + budget: { maxIterations: 1, maxTokens: 100 }, + runtime: 'router', + identity, + seq: 1, + at, + }, + { + kind: 'spawned', + id: `${root}:cancelled`, + parent: root, + label: 'cancelled', + key: 'cancelled-key', + budget: { maxIterations: 1, maxTokens: 100 }, + runtime: 'router', + identity, + seq: 2, + at, + }, + ] + for (const event of spawns) await journal.appendEvent(root, event) + + const out = { answer: 42 } + const outRef = contentAddress(out) + await blobs.put(outRef, out) + await journal.appendEvent(root, { + kind: 'settled', + id: `${root}:done`, + status: 'done', + outRef, + spent, + seq: 3, + at, + }) + await journal.appendEvent(root, { + kind: 'settled', + id: `${root}:down`, + status: 'down', + spent, + seq: 4, + at, + }) + await journal.appendEvent(root, { + kind: 'cancelled', + id: `${root}:cancelled`, + reason: 'stopped', + seq: 5, + at, + }) + + const replayed = await replaySpawnTree(journal, blobs, root) + expect(replayed).toHaveLength(3) + for (const terminal of replayed) { + expect(terminal.handle.identity).toEqual(expectedIdentity) + expect(terminal.handle.identity).not.toBe(identity) + expect(terminal.handle.identity?.correlation).not.toBe(correlation) + expect(Object.isFrozen(terminal.handle)).toBe(true) + expect(Object.isFrozen(terminal.handle.identity)).toBe(true) + expect(Object.isFrozen(terminal.handle.identity?.correlation)).toBe(true) + } + + correlation.experimentId = 'mutated-after-replay' + expect(replayed.map((terminal) => terminal.handle.identity)).toEqual([ + expectedIdentity, + expectedIdentity, + expectedIdentity, + ]) + }) +}) diff --git a/tests/runtime/worktree-cli-executor.test.ts b/tests/runtime/worktree-cli-executor.test.ts index e5d4435d..2c425d49 100644 --- a/tests/runtime/worktree-cli-executor.test.ts +++ b/tests/runtime/worktree-cli-executor.test.ts @@ -28,8 +28,15 @@ vi.mock('node:http', async () => { end: () => { const payload = JSON.parse(body || '{}') as Record if (!bridgeHttpHandler) throw new Error('bridgeHttpHandler not set') - const res = bridgeHttpHandler(payload) as Readable & { statusCode?: number } + const res = bridgeHttpHandler(payload) as Readable & { + statusCode?: number + headers?: Record + } res.statusCode = res.statusCode ?? 200 + res.headers = { + 'x-run-id': String(payload.run_id), + 'x-run-request-digest': `sha256:${'d'.repeat(64)}`, + } cb(res) }, on: () => {}, @@ -105,6 +112,7 @@ const reproducibleCodexProfile: AgentProfile = { function bridgeSseResponse(content: string): Readable { const payload = [ + 'id: 1', `data: ${JSON.stringify({ choices: [{ delta: { content } }], usage: { prompt_tokens: 3, completion_tokens: 5, cost: 0.02 }, @@ -273,6 +281,25 @@ describe('createWorktreeCliExecutor', () => { ).toThrow(/requires codexReproducible/) }) + it('refuses an unsupported profile before creating a worktree', async () => { + const state = freshGitState() + expect(() => + createWorktreeCliExecutor({ + repoRoot: '/workspace', + profile: { + ...authoredProfile, + connections: [{ connectionId: 'github', capabilities: ['issues:read'] }], + }, + harness: 'claude', + taskPrompt: 'x', + runGit: makeFakeGit(state), + runHarness: vi.fn(), + }), + ).toThrow(/profile cannot be materialized.*connections/) + expect(state.worktreesCreated).toEqual([]) + expect(state.worktreesRemoved).toEqual([]) + }) + it('meters reproducible Codex usage and surfaces isolation evidence', async () => { const state = freshGitState() let seen: RunLocalHarnessOptions | undefined @@ -475,6 +502,34 @@ describe('createWorktreeCliExecutor', () => { expect(exec.budgetExempt).toBe(false) }) + it('runs the execute task without requiring a fixed configured prompt', async () => { + const state = freshGitState() + let seen: RunLocalHarnessOptions | undefined + const executor = createWorktreeCliExecutor({ + repoRoot: '/workspace', + profile: authoredProfile, + harness: 'claude', + runGit: makeFakeGit(state), + runHarness: vi.fn(async (options) => { + seen = options + return { + exitCode: 0, + stdout: 'done', + stderr: '', + killedBySignal: null, + durationMs: 1, + timedOut: false, + } + }), + }) + const authorizedTask = { experimentId: 'exp-7', instruction: 'fix the authorized defect' } + + await executor.execute(authorizedTask, new AbortController().signal) + + const prompt = seen?.invocation?.args.find((arg) => arg.includes('exp-7')) + expect(prompt).toContain(JSON.stringify(authorizedTask)) + }) + it('resultArtifact() before execute() resolves throws (fail loud, no fabricated artifact)', () => { const exec = createWorktreeCliExecutor({ repoRoot: '/workspace', @@ -594,7 +649,6 @@ describe('createWorktreeCliExecutor', () => { const factory = createExecutor({ backend: 'cli-worktree', repoRoot: '/workspace', - taskPrompt: 'implement the feature', runId: 'run-live', bridge: { bridgeUrl: 'http://bridge.test', @@ -611,9 +665,13 @@ describe('createWorktreeCliExecutor', () => { }) const spec: AgentSpec = { profile: authoredProfile, harness: null } const exec = factory(spec, { signal: new AbortController().signal, seams: {} }) + const authorizedTask = { + experimentId: 'bridge-exp-9', + instruction: 'implement the authorized feature', + } exec.deliver?.({ steer: 'also update docs' }) - const run = exec.execute(undefined, new AbortController().signal) + const run = exec.execute(authorizedTask, new AbortController().signal) await drain(run as AsyncIterable) const worktreePath = state.worktreesCreated[0] @@ -621,9 +679,9 @@ describe('createWorktreeCliExecutor', () => { expect(requests).toHaveLength(1) expect(requests[0]?.cwd).toBe(worktreePath) expect(requests[0]?.session_id).toBe('session-live') - expect(requests[0]?.messages?.some((m) => m.content.includes('implement the feature'))).toBe( - true, - ) + expect( + requests[0]?.messages?.some((m) => m.content.includes(JSON.stringify(authorizedTask))), + ).toBe(true) expect(requests[0]?.messages?.some((m) => m.content.includes('also update docs'))).toBe(true) expect(checks).toEqual([{ command: 'pnpm test', cwd: worktreePath }]) @@ -642,7 +700,7 @@ describe('createWorktreeCliExecutor', () => { expect(state.worktreesRemoved).toEqual(state.worktreesCreated) }) - it('fails loud on a missing repoRoot / harness / taskPrompt', () => { + it('fails loud on a missing repoRoot, harness, or both task sources', async () => { expect(() => createWorktreeCliExecutor({ repoRoot: '', @@ -659,5 +717,14 @@ describe('createWorktreeCliExecutor', () => { taskPrompt: '', }), ).toThrow(/taskPrompt required/) + + const noTask = createWorktreeCliExecutor({ + repoRoot: '/workspace', + profile: authoredProfile, + harness: 'claude', + }) + await expect(noTask.execute(undefined, new AbortController().signal)).rejects.toThrow( + /execute task required/, + ) }) }) From ca171862b4286a0c05a54e439ba242e05257017d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 05:38:43 -0600 Subject: [PATCH 03/12] fix(supervise): preserve durable run identity --- src/durable/spawn-journal.ts | 4 +- src/mcp/tools/coordination.ts | 29 +++- src/runtime/supervise/scope.ts | 8 +- src/runtime/supervise/supervisor.ts | 21 ++- src/runtime/supervise/types.ts | 3 + tests/kernel/coordination.test.ts | 60 ++++++++ .../spawn-journal-replay-identity.test.ts | 128 +++++++++++++++++- tests/runtime/supervisor-resume.test.ts | 20 +-- 8 files changed, 258 insertions(+), 15 deletions(-) diff --git a/src/durable/spawn-journal.ts b/src/durable/spawn-journal.ts index 3fe71f7d..ffd48c46 100644 --- a/src/durable/spawn-journal.ts +++ b/src/durable/spawn-journal.ts @@ -451,7 +451,9 @@ export async function replaySpawnTree( settled.push({ kind: 'down', handle: handleFor(ev.id, 'failed'), - reason: ev.verdict?.notes ?? 'child down', + // `reason` is written by every current scope. The verdict fallback preserves the + // pre-field convention and the generic text keeps still-older reasonless journals usable. + reason: ev.reason ?? ev.verdict?.notes ?? 'child down', infra: ev.infra === true, restartCount: 0, ...settlementTime(ev.at), diff --git a/src/mcp/tools/coordination.ts b/src/mcp/tools/coordination.ts index 714220b5..b4f2311a 100644 --- a/src/mcp/tools/coordination.ts +++ b/src/mcp/tools/coordination.ts @@ -359,7 +359,11 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin // slot, so the fence must not hold it back. `keyByWorker` is what lets a settlement find its key. const completedKeys = new Set() const keyByWorker = new Map() - let unkeyedAssignmentOrdinal = 0 + // `Scope` advances its node/cursor ordinals on resume, but assignment identity belongs to this + // manager. Seed it independently from durable evidence so a restarted manager never calls new + // work `ordinal:0` when a prior process already used that assignment. Use the maximum rather + // than the number of rows: journals can contain gaps, keyed assignments, and legacy/custom ids. + let unkeyedAssignmentOrdinal = nextUnkeyedAssignmentOrdinal(opts.scope) for (const [key, prior] of opts.scope.resume?.keys ?? []) { if (prior.state === 'completed') completedKeys.add(key) } @@ -1458,6 +1462,29 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin } } +function nextUnkeyedAssignmentOrdinal(scope: Scope): number { + let next = 0 + const views = [scope.resume?.view, scope.view] + for (const view of views) { + if (view === undefined) continue + for (const node of view.nodes) { + const match = /^ordinal:(\d+)$/.exec(node.assignmentId ?? '') + if (match === null) continue + const ordinal = Number(match[1]) + if (!Number.isSafeInteger(ordinal)) { + throw new Error( + `coordination: durable assignment id '${node.assignmentId}' exceeds the safe ordinal range`, + ) + } + next = Math.max(next, ordinal + 1) + } + } + if (!Number.isSafeInteger(next)) { + throw new Error('coordination: durable assignment ordinal space is exhausted') + } + return next +} + function deepFreezeDetached(value: T): T { return deepFreeze(structuredClone(value)) } diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index c64c4604..f169841c 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -1434,6 +1434,7 @@ async function finalizeSettlement( status: 'down', spent: child.spent, infra: settlement.infra, + reason: settlement.reason, seq, at, }) @@ -1759,7 +1760,12 @@ async function runChild( const reconcileError = reconcileOnce(accounting?.reservation ?? live.spent) // A crashed driver child still re-homes the partial inference it durably metered. return downRecord( - errMessage(teardownError ?? reconcileError ?? err), + // The operation that failed is the causal error. Cleanup and accounting can independently + // fail while handling it, but must never replace it with a secondary diagnostic (for + // example, missing terminal usage after a provider crash). Their presence still marks the + // settlement as infrastructure-related, while the unknown spend flags retain the accounting + // failure itself in the durable record. + errMessage(err), teardownError !== undefined || reconcileError !== undefined || aborted || isInfraError(err), executor.metered?.(), ) diff --git a/src/runtime/supervise/supervisor.ts b/src/runtime/supervise/supervisor.ts index 3129df57..b54657af 100644 --- a/src/runtime/supervise/supervisor.ts +++ b/src/runtime/supervise/supervisor.ts @@ -397,7 +397,13 @@ export function createSupervisor(): Supervisor { // starting over. An empty/absent tree — and every run that did NOT opt in — takes the fresh // path unchanged. This wires the already-built+tested resume primitives // (`replaySpawnTree`/`materializeTreeView`); the journal/blob store decide durability. - const prior = opts.resume === true ? await opts.journal.loadTree(opts.runId) : undefined + const existing = await opts.journal.loadTree(opts.runId) + if (opts.resume !== true && existing !== undefined) { + throw new RuntimeRunStateError( + `supervisor: runId '${opts.runId}' already exists; pass resume: true to continue it or use a new runId`, + ) + } + const prior = opts.resume === true ? existing : undefined const resuming = prior !== undefined && prior.length > 0 let resumeFrom: ResumeFrom | undefined let pool: BudgetPool @@ -658,7 +664,7 @@ export function createSupervisor(): Supervisor { const { childWork, driverInference } = await spentFromJournal(journal, opts.runId) return { kind: 'no-winner', - reason: classifyNoWinner(controller, pool, opts, breaker, deadlineExceeded), + reason: classifyNoWinner(controller, pool, opts, breaker, deadlineExceeded, tree), tree, downCount: breaker.downCount(), spentTotal: addSpend(childWork, driverInference), @@ -911,6 +917,7 @@ function classifyNoWinner( opts: SupervisorOpts, breaker: IntensityBreaker, deadlineExceeded: boolean, + tree: TreeView, ): NoWinnerReason { // A tripped breaker is the most specific cause (children kept dying), so it outranks // the abort it raised. A deadline that won the abort race remains budget exhaustion; @@ -919,10 +926,20 @@ function classifyNoWinner( if (breaker.tripped()) return 'all-children-down' if (deadlineExceeded) return 'budget-exhausted' if (controller.signal.aborted) return 'aborted' + // Unknown terminal usage correctly seals the remaining pool, but that accounting consequence is + // not the cause of a run where every child failed. Preserve the lifecycle outcome before asking + // whether any budget channel is now unavailable. A real admission failure with no failed child + // still classifies as budget exhaustion below. + if (allSpawnedChildrenDown(tree)) return 'all-children-down' if (poolExhausted(pool, opts)) return 'budget-exhausted' return 'all-children-down' } +function allSpawnedChildrenDown(tree: TreeView): boolean { + const children = tree.nodes.filter((node) => node.id !== tree.root) + return children.length > 0 && children.every((node) => node.status === 'failed') +} + function poolExhausted(pool: BudgetPool, opts: SupervisorOpts): boolean { const r = pool.readout() if (r.iterationsLeft <= 0) return true diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 709d1865..5669d807 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -792,6 +792,9 @@ export type SpawnEvent = verdict?: DefaultVerdict spent: Spend infra?: boolean + /** Exact child failure. Present on every new `status: 'down'` record; optional only so + * journals written before this field existed remain replayable. */ + reason?: string seq: number at: string } diff --git a/tests/kernel/coordination.test.ts b/tests/kernel/coordination.test.ts index 94d12530..4da07e32 100644 --- a/tests/kernel/coordination.test.ts +++ b/tests/kernel/coordination.test.ts @@ -92,6 +92,66 @@ describe('coordination tools', () => { }) }) + it('continues unkeyed assignment identity past the highest durable resume ordinal', async () => { + const { scope } = mockScope() + Object.defineProperty(scope, 'resume', { + value: { + settled: [], + view: { + root: 'root', + nodes: [ + { + id: 'prior-0', + parent: 'root', + label: 'prior 0', + status: 'done', + runtime: 'router', + budget: { maxIterations: 1, maxTokens: 10 }, + assignmentId: 'ordinal:0', + spent: zeroSpend(), + }, + { + id: 'prior-3', + parent: 'root', + label: 'prior 3', + status: 'done', + runtime: 'router', + budget: { maxIterations: 1, maxTokens: 10 }, + assignmentId: 'ordinal:3', + spent: zeroSpend(), + }, + { + id: 'prior-keyed', + parent: 'root', + label: 'prior keyed', + status: 'done', + runtime: 'router', + budget: { maxIterations: 1, maxTokens: 10 }, + assignmentId: 'key:control', + spent: zeroSpend(), + }, + ], + inFlight: 0, + waiting: 0, + }, + waits: [], + keys: new Map(), + priorSpend: { childWork: zeroSpend(), driverInference: zeroSpend() }, + }, + }) + const tb = createCoordinationTools({ + scope, + blobs, + makeWorkerAgent, + perWorker: { maxIterations: 1, maxTokens: 10 }, + }) + + expect(await tool(tb, 'spawn_agent').handler({ profile: {}, task: 'new work' })).toMatchObject({ + workerId: 'w0', + assignmentId: 'ordinal:4', + }) + }) + it('spawn_agent fails closed at the maxLiveWorkers cap WITHOUT touching the pool', async () => { // A scope whose live (non-terminal) node set is driven by the spawns we make: each successful // spawn appends a `running` node; nothing settles. The conserved pool always admits, so the diff --git a/tests/runtime/spawn-journal-replay-identity.test.ts b/tests/runtime/spawn-journal-replay-identity.test.ts index 5438be39..ed48f891 100644 --- a/tests/runtime/spawn-journal-replay-identity.test.ts +++ b/tests/runtime/spawn-journal-replay-identity.test.ts @@ -1,13 +1,27 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { canonicalCandidateDigest } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { contentAddress, + FileResultBlobStore, + FileSpawnJournal, InMemoryResultBlobStore, InMemorySpawnJournal, materializeTreeView, replaySpawnTree, } from '../../src/durable/spawn-journal' -import type { NodeExecutionIdentity, SpawnEvent } from '../../src/runtime/supervise/types' +import { createBudgetPool } from '../../src/runtime/supervise/budget' +import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { createScope } from '../../src/runtime/supervise/scope' +import type { + Agent, + AgentSpec, + Executor, + NodeExecutionIdentity, + SpawnEvent, +} from '../../src/runtime/supervise/types' const spent = { iterations: 1, @@ -170,4 +184,116 @@ describe('spawn journal replay identity', () => { expectedIdentity, ]) }) + + it('preserves an exact child failure across a durable restart and still reads legacy records', async () => { + const dir = await mkdtemp(join(tmpdir(), 'spawn-down-replay-')) + try { + const root = 'down-restart' + const exactReason = 'provider lost session at turn 7' + const journalPath = join(dir, 'spawn-journal.jsonl') + const blobPath = join(dir, 'blobs') + const journal = new FileSpawnJournal(journalPath) + const blobs = new FileResultBlobStore(blobPath) + await journal.beginTree(root, new Date(0).toISOString()) + + const executor: Executor = { + runtime: 'router', + async execute() { + throw new Error(exactReason) + }, + async teardown() { + return { destroyed: true } + }, + resultArtifact() { + return { + outRef: 'unreachable', + out: undefined, + spent: { + iterations: 0, + tokens: { input: 0, output: 0 }, + usd: 0, + ms: 0, + }, + } + }, + } + const spec: AgentSpec = { + profile: { name: 'failing worker' }, + harness: null, + executor, + } + const agent = { + name: 'failing worker', + act: async () => undefined, + executorSpec: spec, + } as Agent & { executorSpec: AgentSpec } + const scope = createScope({ + parentId: root, + root, + pool: createBudgetPool({ maxIterations: 1, maxTokens: 10 }, () => 0), + journal, + blobs, + executors: createExecutorRegistry(), + seams: {}, + depth: 0, + maxDepth: 4, + signal: new AbortController().signal, + now: () => 0, + }) + + const spawned = scope.spawn(agent, 'fail exactly once', { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'failure', + }) + expect(spawned.ok).toBe(true) + const live = await scope.next() + expect(live).toMatchObject({ kind: 'down', reason: exactReason }) + + // A new store instance has no in-memory state from the writer: this is the restart boundary. + const restartedJournal = new FileSpawnJournal(journalPath) + const durableEvents = await restartedJournal.loadTree(root) + expect( + durableEvents?.find((event) => event.kind === 'settled' && event.status === 'down'), + ).toMatchObject({ reason: exactReason }) + const replayed = await replaySpawnTree( + restartedJournal, + new FileResultBlobStore(blobPath), + root, + ) + expect(replayed).toEqual([expect.objectContaining({ kind: 'down', reason: exactReason })]) + + // Before the reason field existed, some down records carried only verdict.notes and some + // carried neither. Both remain readable; no migration is required to resume an old run. + const legacy = new InMemorySpawnJournal() + await legacy.beginTree('legacy-down', new Date(0).toISOString()) + await legacy.appendEvent('legacy-down', { + kind: 'spawned', + id: 'legacy-down:s0', + parent: 'legacy-down', + label: 'legacy', + budget: { maxIterations: 1, maxTokens: 10 }, + runtime: 'router', + seq: 0, + at: new Date(0).toISOString(), + }) + await legacy.appendEvent('legacy-down', { + kind: 'settled', + id: 'legacy-down:s0', + status: 'down', + spent, + seq: 0, + at: new Date(0).toISOString(), + }) + const legacyReplay = await replaySpawnTree( + legacy, + new InMemoryResultBlobStore(), + 'legacy-down', + ) + expect(legacyReplay).toEqual([ + expect.objectContaining({ kind: 'down', reason: 'child down' }), + ]) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) }) diff --git a/tests/runtime/supervisor-resume.test.ts b/tests/runtime/supervisor-resume.test.ts index 56711284..fc51197c 100644 --- a/tests/runtime/supervisor-resume.test.ts +++ b/tests/runtime/supervisor-resume.test.ts @@ -158,12 +158,12 @@ describe('supervisor durable resume across a real process kill', () => { expect(finalLabels[2]).toBe('c') }) - it('without `resume`, the same durable stores start a FRESH tree (opt-in, not a default)', { + it('refuses a reused durable runId without `resume` before executing or mutating its tree', { timeout: 120_000, }, async () => { - // The default-safety claim: durability is a store choice, resume is a separate opt-in. A - // supervisor run that does not pass `resume` re-runs everything even against a journal that - // already holds committed work. + // A durable run id is one execution identity. Starting fresh under an existing id would mix + // two executions into one journal, so continuing requires an explicit resume and starting over + // requires a new id. const { createSupervisor } = await import('../../src/runtime/supervise/supervisor') const { InMemoryResultBlobStore, InMemorySpawnJournal } = await import( '../../src/durable/spawn-journal' @@ -191,10 +191,12 @@ describe('supervisor durable resume across a real process kill', () => { } const a = await createSupervisor().run(root, 't', opts) expect(a.kind).toBe('winner') - // A second run on the SAME journal + runId still takes the fresh path (`beginTree` is - // idempotent for an identical `at`), and `Scope.resume` was never populated. - const b = await createSupervisor().run(root, 't', opts) - expect(b.kind).toBe('winner') - expect(acts).toBe(2) + const committed = await journal.loadTree(opts.runId) + + await expect(createSupervisor().run(root, 't', opts)).rejects.toThrow( + /runId 'no-resume' already exists.*resume: true.*new runId/, + ) + expect(acts).toBe(1) + expect(await journal.loadTree(opts.runId)).toEqual(committed) }) }) From 5977ac6cba99a23ca1aa176584d8afeb6b62fcad Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 05:39:07 -0600 Subject: [PATCH 04/12] feat(candidate): export profile conversion --- src/candidate-execution/index.ts | 1 + tests/candidate-bundle-builder.test.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/candidate-execution/index.ts b/src/candidate-execution/index.ts index 3b62c6af..794b4bd7 100644 --- a/src/candidate-execution/index.ts +++ b/src/candidate-execution/index.ts @@ -58,6 +58,7 @@ export { prepareAgentCandidateExecution, } from './prepare' export { + agentCandidateProfileAsAgentProfile, applyExactAgentProfileDiff, assertCandidateProfileBinding, parseExactAgentProfile, diff --git a/tests/candidate-bundle-builder.test.ts b/tests/candidate-bundle-builder.test.ts index f4d5d65c..8228fe50 100644 --- a/tests/candidate-bundle-builder.test.ts +++ b/tests/candidate-bundle-builder.test.ts @@ -15,6 +15,7 @@ import { afterEach, describe, expect, expectTypeOf, it } from 'vitest' import { assertCandidateProfileBinding } from '../src/candidate-execution' import { + agentCandidateProfileAsAgentProfile, type BuildAgentCandidateBundleInput, buildAgentCandidateBundle, sealAgentCandidateBundle, @@ -124,6 +125,10 @@ describe('public agent candidate bundle builder', () => { name: 'review/SKILL.md', sha256: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), }) + expect(agentCandidateProfileAsAgentProfile(first.profile)).toMatchObject({ + name: 'candidate', + harness: 'codex', + }) expect(first.knowledge).toEqual(input.knowledge) const verified = await verifyAgentCandidateBundle(first, { From 8cfebcff7efbeef832b10fdda67164d6a48de7f1 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 06:03:21 -0600 Subject: [PATCH 05/12] fix(supervise): persist structured worker traces --- docs/canonical-api.md | 3 +- package.json | 2 +- pnpm-lock.yaml | 23 +- pnpm-workspace.yaml | 2 +- src/durable/content-address.ts | 16 ++ src/durable/spawn-journal.ts | 44 ++-- src/index.ts | 2 + src/mcp/tools/coordination.ts | 44 +++- src/runtime/index.ts | 9 + src/runtime/supervise/scope.ts | 60 +++++- src/runtime/supervise/trace-evidence.ts | 150 +++++++++++++ src/runtime/supervise/types.ts | 33 +++ tests/kernel/coordination.test.ts | 134 ++++++++++-- tests/runtime/worker-trace-evidence.test.ts | 220 ++++++++++++++++++++ 14 files changed, 693 insertions(+), 49 deletions(-) create mode 100644 src/durable/content-address.ts create mode 100644 src/runtime/supervise/trace-evidence.ts create mode 100644 tests/runtime/worker-trace-evidence.test.ts diff --git a/docs/canonical-api.md b/docs/canonical-api.md index d6aaf8ab..eeb305a9 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -6,7 +6,7 @@ Run pnpm docs:freshness after editing this file. --> > **Version 0.109.1.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> `agent-eval` must satisfy `>=0.135.1 <0.136.0`. +> `agent-eval` must satisfy `>=0.135.3 <0.136.0`. > `sandbox` must satisfy `>=0.15.0 <0.16.0`. > Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.36.0 <0.37.0`. > @@ -94,6 +94,7 @@ A general "loop" primitive is the single most common modelling error in this rep |---|---|---| | Run one product chat turn with streamed events, ordered persistence hooks, and stable execution/turn identity | `handleChatTurn(...)` + `deriveExecutionId(...)`: `/durable`; pass the derived id as both `executionId` and `turnId` on initial dispatch | importing the broad package entry from an edge worker, treating `executionId` alone as dispatch idempotency, or rebuilding framing and persistence ordering in the product | | Run a supervisor toward a goal with default setup | `supervise(profile, task, { budget, backend? })`: `/kernel` | hand-wiring `createSupervisor().run` + `blobs`/`perWorker`/`journal`/`executors`; reaching for lower-level calls before you need a specific counterparty | +| Analyze a settled worker's tool use | `run_analyst` over the worker's persisted `WorkerTraceEvidence`; Runtime reconstructs agent-eval's bounded `TraceAnalysisStore`: `/kernel` + `/mcp` | passing the worker's final prose or result blob to a trace analyst; a worker with no structured tool spans carries an explicit unavailable reason and analysis is refused | | **Supervise agents to solve a graded `AgenticSurface` task** (workers `runAgentic` the surface, settle on its own check, driver self-improves from the failing tests) | `superviseSurface(profile, task, { surface, worker })`: `/kernel` | a worker-seam + a "self-improving supervisor" wrapper around `supervise()`; passing a custom `makeWorkerAgent` that runs `runAgentic` | | Run a profile through a topology shape over the keystone Supervisor, end-to-end | `runPersonified({ persona, shape, task, budget })`: `/kernel` | a hand-rolled `createSupervisor().run` + seam-wiring helper | | Loop a worker over one evolving artifact, K rounds, stop-when-good | `loopUntil(seed, spec)` as the `shape`: `/kernel` | a `while(!done){runWorker();decide()}` hand-loop or "multi-attempt refine driver" | diff --git a/package.json b/package.json index abff23ab..de4e3ed2 100644 --- a/package.json +++ b/package.json @@ -162,7 +162,7 @@ "license": "MIT", "packageManager": "pnpm@11.17.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.135.1 <0.136.0", + "@tangle-network/agent-eval": ">=0.135.3 <0.136.0", "@tangle-network/agent-interface": ">=0.36.0 <0.37.0", "@tangle-network/sandbox": ">=0.15.0 <0.16.0", "playwright": "^1.40.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2cd682b1..05a10da3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,8 +10,8 @@ catalogs: specifier: 0.18.5 version: 0.18.5 '@tangle-network/agent-eval': - specifier: 0.135.1 - version: 0.135.1 + specifier: 0.135.3 + version: 0.135.3 '@tangle-network/agent-interface': specifier: 0.36.0 version: 0.36.0 @@ -62,7 +62,7 @@ importers: version: 2.5.5 '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.135.1 + version: 0.135.3 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.36.0 @@ -113,7 +113,7 @@ importers: dependencies: '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.135.1 + version: 0.135.3 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.36.0 @@ -1116,6 +1116,11 @@ packages: engines: {node: '>=20'} hasBin: true + '@tangle-network/agent-eval@0.135.3': + resolution: {integrity: sha512-KVTXinEzGQJ1fQ0cUm6bKEnnD8gGijb6HiyYG28/8vvndvq5cJsBCuPmcCw2wfvEqIUOlS/3ABDOQbxGIiGJRw==} + engines: {node: '>=20'} + hasBin: true + '@tangle-network/agent-interface@0.36.0': resolution: {integrity: sha512-l48SiAY5Atx6ZTL86/XP2wSUPfJJ1sga82/F6V0KqwVkE5MHWiFql6wsLoqk0lc7s4EJe6+cKky+2zzxFuxDpg==} @@ -2767,6 +2772,16 @@ snapshots: hono: 4.12.32 zod: 4.4.3 + '@tangle-network/agent-eval@0.135.3': + dependencies: + '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) + '@ax-llm/ax': 23.0.5(zod@4.4.3) + '@hono/node-server': 2.0.12(hono@4.12.32) + '@tangle-network/agent-core': 0.4.25 + '@tangle-network/agent-interface': 0.36.0 + hono: 4.12.32 + zod: 4.4.3 + '@tangle-network/agent-interface@0.36.0': dependencies: '@noble/hashes': 1.8.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a5ecb70c..1130e06d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -18,7 +18,7 @@ allowBuilds: catalog: '@arethetypeswrong/cli': 0.18.5 '@types/node': 26.1.1 - '@tangle-network/agent-eval': 0.135.1 + '@tangle-network/agent-eval': 0.135.3 '@tangle-network/agent-interface': 0.36.0 '@tangle-network/agent-knowledge': 6.1.10 '@tangle-network/agent-profile-materialize': 0.9.2 diff --git a/src/durable/content-address.ts b/src/durable/content-address.ts new file mode 100644 index 00000000..6ff89ffb --- /dev/null +++ b/src/durable/content-address.ts @@ -0,0 +1,16 @@ +import { createHash } from 'node:crypto' + +/** Stable content address shared by result and trace artifacts. */ +export function contentAddress(artifact: unknown): string { + const hex = createHash('sha256').update(stableStringify(artifact), 'utf-8').digest('hex') + return `sha256:${hex}` +} + +function stableStringify(value: unknown): string { + if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' + if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]` + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(',')}}` +} diff --git a/src/durable/spawn-journal.ts b/src/durable/spawn-journal.ts index ffd48c46..160510bf 100644 --- a/src/durable/spawn-journal.ts +++ b/src/durable/spawn-journal.ts @@ -20,7 +20,7 @@ * @experimental */ -import { createHash } from 'node:crypto' +import { workerTraceAnalysisStore } from '../runtime/supervise/trace-evidence' import type { NodeExecutionIdentity, NodeId, @@ -36,8 +36,11 @@ import type { } from '../runtime/supervise/types' import type { PendingWait } from '../runtime/supervise/wait' import { zeroTokenUsage } from '../runtime/util' +import { contentAddress } from './content-address' import { parseCommittedJsonLines, prepareJsonlAppend, writeAllBytes } from './jsonl-file' +export { contentAddress } from './content-address' + // ── Content addressing ────────────────────────────────────────────────────── /** @@ -49,20 +52,6 @@ import { parseCommittedJsonLines, prepareJsonlAppend, writeAllBytes } from './js * Stable encoding: object keys are sorted recursively so two structurally-equal * artifacts hash identically regardless of key insertion order. */ -export function contentAddress(artifact: unknown): string { - const hex = createHash('sha256').update(stableStringify(artifact), 'utf-8').digest('hex') - return `sha256:${hex}` -} - -function stableStringify(value: unknown): string { - if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null' - if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]` - const entries = Object.entries(value as Record) - .filter(([, v]) => v !== undefined) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(',')}}` -} - // ── Result blob store ───────────────────────────────────────────────────────── /** @@ -413,6 +402,7 @@ export async function replaySpawnTree( reason: 'wait cancelled', infra: false, restartCount: 0, + trace: { status: 'unavailable', reason: 'not-an-executor' }, ...settlementTime(ev.at), seq: ev.seq, }) @@ -430,6 +420,7 @@ export async function replaySpawnTree( out: outcome, outRef: ev.outRef, spent: zeroSpend(), + trace: { status: 'unavailable', reason: 'not-an-executor' }, ...settlementTime(ev.at), seq: ev.seq, }) @@ -442,12 +433,15 @@ export async function replaySpawnTree( reason: ev.reason, infra: false, restartCount: 0, + trace: { status: 'unavailable', reason: 'execution-did-not-start' }, ...settlementTime(ev.at), seq: ev.seq, }) continue } if (ev.status === 'down') { + const trace = traceEvidenceFor(ev) + if (trace.status === 'available') await workerTraceAnalysisStore(trace, blobs) settled.push({ kind: 'down', handle: handleFor(ev.id, 'failed'), @@ -456,6 +450,7 @@ export async function replaySpawnTree( reason: ev.reason ?? ev.verdict?.notes ?? 'child down', infra: ev.infra === true, restartCount: 0, + trace, ...settlementTime(ev.at), seq: ev.seq, }) @@ -473,6 +468,8 @@ export async function replaySpawnTree( `replaySpawnTree: blob store has no artifact for outRef '${ev.outRef}' (node '${ev.id}', seq ${ev.seq})`, ) } + const trace = traceEvidenceFor(ev) + if (trace.status === 'available') await workerTraceAnalysisStore(trace, blobs) settled.push({ kind: 'done', handle: handleFor(ev.id, 'done'), @@ -480,6 +477,7 @@ export async function replaySpawnTree( outRef: ev.outRef, verdict: ev.verdict, spent: ev.spent, + trace, ...settlementTime(ev.at), seq: ev.seq, }) @@ -595,17 +593,20 @@ export function materializeTreeView(events: SpawnEvent[]): TreeView { node.status = ev.status === 'done' ? 'done' : 'failed' node.spent = ev.spent node.outRef = ev.outRef + node.trace = traceEvidenceFor(ev) const settledAt = Date.parse(ev.at) if (Number.isFinite(settledAt)) node.settledAt = settledAt } else if (ev.kind === 'woken') { const node = requireNode(nodes, ev.id) node.status = ev.by === 'cancelled' ? 'cancelled' : 'done' node.outRef = ev.outRef + node.trace = { status: 'unavailable', reason: 'not-an-executor' } const settledAt = Date.parse(ev.at) if (Number.isFinite(settledAt)) node.settledAt = settledAt } else { const node = requireNode(nodes, ev.id) node.status = 'cancelled' + node.trace = { status: 'unavailable', reason: 'execution-did-not-start' } const settledAt = Date.parse(ev.at) if (Number.isFinite(settledAt)) node.settledAt = settledAt } @@ -675,6 +676,7 @@ interface MutableSnapshot { executionBindings?: NonNullable[number][] spent: Spend outRef?: string + trace?: NodeSnapshot['trace'] settledAt?: number } @@ -717,10 +719,22 @@ function freezeSnapshot(node: MutableSnapshot): NodeSnapshot { node.executionBindings === undefined ? undefined : Object.freeze([...node.executionBindings]), spent: node.spent, outRef: node.outRef, + trace: node.trace, settledAt: node.settledAt, } } +function traceEvidenceFor( + event: Extract, +): NonNullable { + return ( + event.trace ?? { + status: 'unavailable', + reason: 'legacy-settlement-without-trace-evidence', + } + ) +} + function isNoEntError(err: unknown): boolean { return ( typeof err === 'object' && diff --git a/src/index.ts b/src/index.ts index 0094a549..2ee0d210 100644 --- a/src/index.ts +++ b/src/index.ts @@ -215,6 +215,8 @@ export type { SupervisedResult, Supervisor, SupervisorFinalizer, + WorkerTraceEvidence, + WorkerTraceUnavailableReason, } from './runtime' // ── Runtime hooks ──────────────────────────────────────────────────── export type { diff --git a/src/mcp/tools/coordination.ts b/src/mcp/tools/coordination.ts index b4f2311a..67bafc75 100644 --- a/src/mcp/tools/coordination.ts +++ b/src/mcp/tools/coordination.ts @@ -9,6 +9,7 @@ */ import { randomUUID } from 'node:crypto' +import type { TraceAnalysisStore } from '@tangle-network/agent-eval' import { type AgentProfile, agentProfileSchema, @@ -25,12 +26,14 @@ import type { Settled, Spend, Agent as SuperviseAgent, + WorkerTraceEvidence, } from '../../runtime' import { assertValidBudget } from '../../runtime/supervise/budget' import { type WatchTraceOptions, watchTrace } from '../../runtime/supervise/detector-monitor' import { freeSlots } from '../../runtime/supervise/dispatch' import { type BusRecord, type BusStats, createEventBus } from '../../runtime/supervise/event-bus' import type { WorkerProgress } from '../../runtime/supervise/progress' +import { workerTraceAnalysisStore } from '../../runtime/supervise/trace-evidence' import type { McpToolDescriptor } from '../server' /** A worker the driver has drained via `await_event`. */ @@ -51,6 +54,8 @@ export interface SettledWorker { readonly valid?: boolean readonly outRef?: string readonly reason?: string + /** Structured tool-call evidence, never the worker's final prose. */ + readonly trace: WorkerTraceEvidence /** True when projected from a prior process of the same durable run. */ readonly resumed?: boolean /** Epoch ms from the durable terminal record — the resolution a progress-based stop rule needs @@ -92,7 +97,7 @@ export type QuestionPolicy = 'auto' | 'mustDecide' | 'bubble' | 'failClosed' export interface AnalystRegistry { readonly kinds: ReadonlyArray<{ id: string; description: string; area: string }> - readonly run: (kindId: string, trace: unknown) => Promise + readonly run: (kindId: string, trace: TraceAnalysisStore) => Promise } /** A trace-analyst result re-entered as a message on the bus (the `finding` event kind). */ @@ -379,6 +384,13 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin const materialization = settled.handle.materialization ?? node?.materialization const executionBindings = settled.handle.executionBindings ?? node?.executionBindings const settledAt = settled.settledAt ?? node?.settledAt + const trace = + settled.trace ?? + node?.trace ?? + ({ + status: 'unavailable', + reason: 'legacy-settlement-without-trace-evidence', + } as const) const common = { id: settled.handle.id, ...(assignmentId === undefined ? {} : { assignmentId }), @@ -386,6 +398,7 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin ...(materialization === undefined ? {} : { materialization }), ...(executionBindings === undefined ? {} : { executionBindings }), ...(settledAt === undefined ? {} : { settledAt }), + trace, ...(resumed ? { resumed: true as const } : {}), } return deepFreezeDetached( @@ -537,11 +550,11 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin if ( pending.analyze && pending.worker.status === 'done' && - pending.worker.outRef && + pending.worker.trace.status === 'available' && opts.analysts && opts.analyzeOnSettle?.length ) { - const trace = await opts.blobs.get(pending.worker.outRef) + const trace = await workerTraceAnalysisStore(pending.worker.trace, opts.blobs) for (const analyst of opts.analyzeOnSettle) { const findings = await opts.analysts.run(analyst, trace) await bus.publish({ @@ -863,6 +876,7 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin spent: node.spent, ...(node.settledAt === undefined ? {} : { settledAt: node.settledAt }), ...(node.outRef === undefined ? {} : { outRef: node.outRef }), + ...(node.trace === undefined ? {} : { trace: node.trace }), ...(resumed ? { resumed: true } : {}), }) @@ -1438,12 +1452,28 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin handler: async (raw) => { const a = obj(raw) const id = str(a.workerId, 'workerId') - const node = opts.scope.view.nodes.find((n) => n.id === id) + const node = nodeForWorker(id) if (!node) return { error: `unknown workerId ${JSON.stringify(id)}` } - if (!node.outRef) + if (isLive(node.status)) { return { error: `worker ${JSON.stringify(id)} has not settled — no trace to analyze yet` } - const trace = await opts.blobs.get(node.outRef) - return { findings: await opts.analysts?.run(str(a.kind, 'kind'), trace) } + } + const trace = + ledger.find((worker) => worker.id === id)?.trace ?? + node.trace ?? + ({ + status: 'unavailable', + reason: 'legacy-settlement-without-trace-evidence', + } as const) + let store: TraceAnalysisStore + try { + store = await workerTraceAnalysisStore(trace, opts.blobs) + } catch (error) { + return { + error: error instanceof Error ? error.message : String(error), + trace, + } + } + return { findings: await opts.analysts?.run(str(a.kind, 'kind'), store) } }, }) } diff --git a/src/runtime/index.ts b/src/runtime/index.ts index bcf3f4a5..e74ee3d8 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -715,6 +715,13 @@ export { type SupervisorToolDescriptor, supervisorAgent, } from './supervise/supervisor-agent' +export { + captureWorkerTraceEvidence, + parseWorkerToolTraceArtifact, + WORKER_TOOL_TRACE_SCHEMA_VERSION, + type WorkerToolTraceArtifact, + workerTraceAnalysisStore, +} from './supervise/trace-evidence' // The substrate-agnostic trace source: a worker's tool calls as agent-eval `ToolSpan`s, from an // OWNED loop (push) OR a sandbox box session (message parts). The common currency for both analysts. export { @@ -776,6 +783,8 @@ export type { UsageEvent, WaitOpts, WidenGate, + WorkerTraceEvidence, + WorkerTraceUnavailableReason, } from './supervise/types' // WAIT-STATES: a tree node that waits on wall-clock time (`timer`) or a named external predicate // (`poll`) with NO executor, NO sandbox, and NO conserved budget — journaled with its absolute diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index f169841c..dbe400b1 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -60,6 +60,7 @@ import { type WorkerProgress, } from './progress' import { detachedSnapshot } from './snapshot' +import { captureWorkerTraceEvidence } from './trace-evidence' import type { TraceSource } from './trace-source' import type { Agent, @@ -92,6 +93,7 @@ import type { TreeView, UsageEvent, WaitOpts, + WorkerTraceEvidence, } from './types' import { assertWaitWithinDeadline, @@ -209,6 +211,8 @@ interface LiveChild { readonly key?: string spent: Spend outRef?: string + /** Durable structured tool evidence once this executor is terminal. */ + trace?: WorkerTraceEvidence /** Exact terminal timestamp committed to the journal. */ settledAt?: number /** Resolves with the terminal settlement WITHOUT a `seq` — `next()` stamps the seq. */ @@ -259,6 +263,7 @@ type PreSeqSettled = outRef: string verdict?: DefaultVerdict spent: Spend + trace: WorkerTraceEvidence /** A driver child's OWN-inference subtree total (from `Executor.metered()`) — journaled as a * `metered` event for this node, NOT reconciled (already debited live via `observe`). */ metered?: Spend @@ -268,6 +273,7 @@ type PreSeqSettled = reason: string infra: boolean restartCount: number + trace: WorkerTraceEvidence /** A CRASHED driver child's partial OWN-inference subtree total — re-homed on the down path * too, so the journal matches the pool (which already debited it via `observe`). */ metered?: Spend @@ -992,15 +998,33 @@ export function createScope(args: ScopeArgs): Scope { live.executorDone = true live.lastActivityAt = now() if (resolution.kind === 'cancelled') { - return { kind: 'down', reason: resolution.reason, infra: false, restartCount: 0 } + return { + kind: 'down', + reason: resolution.reason, + infra: false, + restartCount: 0, + trace: { status: 'unavailable', reason: 'not-an-executor' }, + } } const outRef = contentAddress(resolution.outcome) await args.blobs.put(outRef, resolution.outcome) - return { kind: 'done', out: resolution.outcome, outRef, spent: zeroSpend() } + return { + kind: 'done', + out: resolution.outcome, + outRef, + spent: zeroSpend(), + trace: { status: 'unavailable', reason: 'not-an-executor' }, + } }) .catch((err): PreSeqSettled => { live.executorDone = true - return { kind: 'down', reason: errMessage(err), infra: true, restartCount: 0 } + return { + kind: 'down', + reason: errMessage(err), + infra: true, + restartCount: 0, + trace: { status: 'unavailable', reason: 'not-an-executor' }, + } }) .then((s) => { live.resolved = s @@ -1428,6 +1452,7 @@ async function finalizeSettlement( const at = new Date(settledAt).toISOString() if (settlement.kind === 'down') { child.status = 'failed' + child.trace = settlement.trace await args.journal.appendEvent(args.root, { kind: 'settled', id: child.id, @@ -1435,6 +1460,7 @@ async function finalizeSettlement( spent: child.spent, infra: settlement.infra, reason: settlement.reason, + trace: settlement.trace, seq, at, }) @@ -1475,6 +1501,7 @@ async function finalizeSettlement( reason: settlement.reason, infra: settlement.infra, restartCount: settlement.restartCount, + trace: settlement.trace, settledAt, seq, } @@ -1483,6 +1510,7 @@ async function finalizeSettlement( child.status = 'done' child.outRef = settlement.outRef child.spent = settlement.spent + child.trace = settlement.trace await args.journal.appendEvent(args.root, { kind: 'settled', id: child.id, @@ -1490,6 +1518,7 @@ async function finalizeSettlement( outRef: settlement.outRef, ...(settlement.verdict ? { verdict: settlement.verdict } : {}), spent: settlement.spent, + trace: settlement.trace, seq, at, }) @@ -1533,6 +1562,7 @@ async function finalizeSettlement( outRef: settlement.outRef, ...(settlement.verdict ? { verdict: settlement.verdict } : {}), spent: settlement.spent, + trace: settlement.trace, settledAt, seq, } @@ -1571,6 +1601,7 @@ async function finalizeWait( reason: settlement.reason, infra: settlement.infra, restartCount: settlement.restartCount, + trace: settlement.trace, settledAt, seq, } @@ -1606,6 +1637,7 @@ async function finalizeWait( out: settlement.out as Out, outRef: settlement.outRef, spent: settlement.spent, + trace: settlement.trace, settledAt, seq, } @@ -1639,6 +1671,11 @@ async function runChild( let started = false let terminalTelemetryCaptured = false let teardownStarted = false + let traceEvidence: WorkerTraceEvidence | undefined + const captureTraceOnce = async (): Promise => { + traceEvidence ??= await captureWorkerTraceEvidence(live.readTraceSource, blobs, started) + return traceEvidence + } const teardownOnce = async (grace: number | 'brutalKill' | 'infinity'): Promise => { if (teardownStarted) return teardownStarted = true @@ -1711,10 +1748,11 @@ async function runChild( // A driver child's OWN-inference subtree total — re-homed by the parent on EVERY settle exit // (done, aborted, crash) so the journal always matches what the pool already debited. const ownMetered = executor.metered?.() + const trace = await captureTraceOnce() if (childAbort.signal.aborted) { await teardownOnce(opts.shutdown ?? 'brutalKill') - return downRecord('aborted before settle', true, ownMetered) + return downRecord('aborted before settle', true, trace, ownMetered) } // The durable record is keyed by the canonical content address of the output — the @@ -1732,6 +1770,7 @@ async function runChild( outRef, ...(artifact.verdict ? { verdict: artifact.verdict } : {}), spent: live.spent, + trace, ...(ownMetered ? { metered: ownMetered } : {}), } } catch (err) { @@ -1740,6 +1779,8 @@ async function runChild( live.executorDone = true // A recursive executor can still report the nested work committed before it threw. // Reconcile that whole partial subtree while journaling its child-work component separately. + // A box-backed trace must be collected before teardown destroys the session that owns it. + const trace = await captureTraceOnce() let teardownError: unknown try { await teardownOnce('brutalKill') @@ -1767,6 +1808,7 @@ async function runChild( // failure itself in the durable record. errMessage(err), teardownError !== undefined || reconcileError !== undefined || aborted || isInfraError(err), + trace, executor.metered?.(), ) } @@ -1851,6 +1893,7 @@ function makeTreeView(root: NodeId, children: Map): TreeView spent: c.spent, ...(c.settledAt === undefined ? {} : { settledAt: c.settledAt }), ...(c.outRef ? { outRef: c.outRef } : {}), + ...(c.trace ? { trace: c.trace } : {}), })) return { root, @@ -2042,8 +2085,13 @@ function abortError(signal: AbortSignal): Error { return error } -function downRecord(reason: string, infra: boolean, metered?: Spend): PreSeqSettled { - return { kind: 'down', reason, infra, restartCount: 0, ...(metered ? { metered } : {}) } +function downRecord( + reason: string, + infra: boolean, + trace: WorkerTraceEvidence, + metered?: Spend, +): PreSeqSettled { + return { kind: 'down', reason, infra, restartCount: 0, trace, ...(metered ? { metered } : {}) } } function zeroSpend(): Spend { diff --git a/src/runtime/supervise/trace-evidence.ts b/src/runtime/supervise/trace-evidence.ts new file mode 100644 index 00000000..36b6a9f1 --- /dev/null +++ b/src/runtime/supervise/trace-evidence.ts @@ -0,0 +1,150 @@ +/** + * Durable structured tool evidence for supervised workers. + * + * The executor-owned `TraceSource` is collected before teardown, snapshotted into the shared + * content-addressed blob store, and only then referenced by a settlement. Analysis reconstructs + * agent-eval's bounded `TraceAnalysisStore` from those exact bytes. Worker output is never accepted + * as a substitute for tool evidence. + * + * @experimental + */ + +import { + isToolSpan, + type Span, + type ToolSpan, + type TraceAnalysisStore, + toolSpansToTraceAnalysisStore, +} from '@tangle-network/agent-eval' +import { contentAddress } from '../../durable/content-address' +import { ValidationError } from '../../errors' +import type { TraceSource } from './trace-source' +import type { ResultBlobStore, WorkerTraceEvidence } from './types' + +/** Schema version for content-addressed worker tool-trace artifacts. */ +export const WORKER_TOOL_TRACE_SCHEMA_VERSION = 1 as const + +/** Bytes stored under `WorkerTraceEvidence.traceRef`. */ +export interface WorkerToolTraceArtifact { + readonly schemaVersion: typeof WORKER_TOOL_TRACE_SCHEMA_VERSION + readonly spans: ReadonlyArray +} + +/** Collect and persist one executor's structured tool trace without changing its task outcome. */ +export async function captureWorkerTraceEvidence( + readSource: (() => TraceSource | undefined) | undefined, + blobs: ResultBlobStore, + executed: boolean, +): Promise { + if (!executed) return unavailable('execution-did-not-start') + if (readSource === undefined) return unavailable('executor-did-not-expose-trace-source') + + let source: TraceSource | undefined + try { + source = readSource() + } catch { + return unavailable('trace-source-unavailable') + } + if (source === undefined) return unavailable('trace-source-unavailable') + + let collected: unknown + try { + collected = await source.collect() + } catch { + return unavailable('trace-collection-failed') + } + if (!Array.isArray(collected) || collected.some((span) => !isToolSpanValue(span))) { + return unavailable('invalid-tool-spans') + } + if (collected.length === 0) return unavailable('no-tool-spans-captured') + + let artifact: WorkerToolTraceArtifact + let traceRef: string + try { + const snapshot = structuredClone(collected) as ToolSpan[] + // Use agent-eval's canonical adapter as the full integrity check before the durable receipt + // claims these spans are analyzable. `isToolSpan` only narrows the discriminated union. + toolSpansToTraceAnalysisStore(snapshot) + artifact = { + schemaVersion: WORKER_TOOL_TRACE_SCHEMA_VERSION, + spans: snapshot, + } + traceRef = contentAddress(artifact) + } catch { + return unavailable('invalid-tool-spans') + } + try { + await blobs.put(traceRef, artifact) + } catch { + return unavailable('trace-persistence-failed') + } + return Object.freeze({ status: 'available', traceRef, spanCount: artifact.spans.length }) +} + +/** Rehydrate exact persisted spans through agent-eval's one bounded trace-analysis adapter. */ +export async function workerTraceAnalysisStore( + evidence: WorkerTraceEvidence, + blobs: Pick, +): Promise { + if (evidence.status === 'unavailable') { + // The published adapter owns the missing-evidence error and its classification. + return toolSpansToTraceAnalysisStore(undefined) + } + const raw = await blobs.get(evidence.traceRef) + if (raw === undefined) { + throw new ValidationError( + `worker trace blob '${evidence.traceRef}' is missing; the settlement evidence is incomplete`, + ) + } + const artifact = parseWorkerToolTraceArtifact(raw, evidence.traceRef) + if (artifact.spans.length !== evidence.spanCount) { + throw new ValidationError( + `worker trace blob '${evidence.traceRef}' has ${artifact.spans.length} spans but its settlement records ${evidence.spanCount}`, + ) + } + return toolSpansToTraceAnalysisStore(artifact.spans) +} + +/** Validate a stored trace artifact before an analyst or replay trusts it. */ +export function parseWorkerToolTraceArtifact( + value: unknown, + traceRef = '', +): WorkerToolTraceArtifact { + if ( + value === null || + typeof value !== 'object' || + (value as { schemaVersion?: unknown }).schemaVersion !== WORKER_TOOL_TRACE_SCHEMA_VERSION || + !Array.isArray((value as { spans?: unknown }).spans) + ) { + throw new ValidationError( + `worker trace blob '${traceRef}' is not a version-${WORKER_TOOL_TRACE_SCHEMA_VERSION} tool trace artifact`, + ) + } + const spans = (value as { spans: unknown[] }).spans + if (spans.length === 0) { + // Missing spans cannot prove a tool-free run; preserve agent-eval's explicit integrity error. + return { + schemaVersion: WORKER_TOOL_TRACE_SCHEMA_VERSION, + spans: spans as ToolSpan[], + } + } + if (spans.some((span) => !isToolSpanValue(span))) { + throw new ValidationError(`worker trace blob '${traceRef}' contains a non-tool span`) + } + return { + schemaVersion: WORKER_TOOL_TRACE_SCHEMA_VERSION, + spans: spans as ToolSpan[], + } +} + +function unavailable(reason: Extract['reason']) { + return Object.freeze({ status: 'unavailable' as const, reason }) +} + +function isToolSpanValue(value: unknown): value is ToolSpan { + try { + return isToolSpan(value as Span) + } catch { + return false + } +} diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 5669d807..604e0923 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -165,6 +165,31 @@ export interface Executor { metered?(): Spend | undefined } +/** Why Runtime cannot provide structured tool-call evidence for one settled execution. */ +export type WorkerTraceUnavailableReason = + | 'execution-did-not-start' + | 'executor-did-not-expose-trace-source' + | 'trace-source-unavailable' + | 'no-tool-spans-captured' + | 'invalid-tool-spans' + | 'trace-collection-failed' + | 'trace-persistence-failed' + | 'legacy-settlement-without-trace-evidence' + | 'not-an-executor' + +/** Durable proof of a worker's structured tool trace, or the exact reason it is unavailable. */ +export type WorkerTraceEvidence = + | { + readonly status: 'available' + /** Content-addressed pointer to a persisted `WorkerToolTraceArtifact`. */ + readonly traceRef: string + readonly spanCount: number + } + | { + readonly status: 'unavailable' + readonly reason: WorkerTraceUnavailableReason + } + /** Split used by a recursive executor when journaled child work differs from the full amount * reconciled against its parent reservation. */ export interface ExecutorAccounting { @@ -516,6 +541,8 @@ export type Settled = outRef: string verdict?: DefaultVerdict spent: Spend + /** Structured tool evidence captured before this settlement was journaled. */ + trace: WorkerTraceEvidence /** Epoch ms parsed from the durable settlement record when available. */ settledAt?: number seq: number @@ -527,6 +554,8 @@ export type Settled = /** True = infrastructure failure (excluded from merge `n` / equal-k), not a bad result. */ infra: boolean restartCount: number + /** Partial structured tool evidence captured before this failure was journaled. */ + trace: WorkerTraceEvidence /** Epoch ms parsed from the durable settlement/cancellation record when available. */ settledAt?: number seq: number @@ -730,6 +759,8 @@ export interface NodeSnapshot { readonly spent: Spend /** `outRef` once the node is `done` (the replay/result pointer). */ readonly outRef?: string + /** Present on terminal executor nodes; legacy records carry an explicit unavailable reason. */ + readonly trace?: WorkerTraceEvidence } /** The live tree — what `scope.view` / `RootHandle.view()` materialize for a viewer. */ @@ -795,6 +826,8 @@ export type SpawnEvent = /** Exact child failure. Present on every new `status: 'down'` record; optional only so * journals written before this field existed remain replayable. */ reason?: string + /** Structured tool evidence. Optional only for journals written before trace capture. */ + trace?: WorkerTraceEvidence seq: number at: string } diff --git a/tests/kernel/coordination.test.ts b/tests/kernel/coordination.test.ts index 4da07e32..2ee1bdb1 100644 --- a/tests/kernel/coordination.test.ts +++ b/tests/kernel/coordination.test.ts @@ -1,11 +1,65 @@ +import type { ToolSpan, TraceAnalysisStore } from '@tangle-network/agent-eval' import { describe, expect, it } from 'vitest' import { createMcpServer } from '../../src/mcp/server' import { type CoordinationEvent, createCoordinationTools } from '../../src/mcp/tools/coordination' -import type { Agent, ResultBlobStore, Scope, Spend } from '../../src/runtime' -import { createPushTraceSource, watchTrace } from '../../src/runtime' +import type { Agent, ResultBlobStore, Scope, Spend, WorkerTraceEvidence } from '../../src/runtime' +import { + contentAddress, + createPushTraceSource, + WORKER_TOOL_TRACE_SCHEMA_VERSION, + watchTrace, +} from '../../src/runtime' const zeroSpend = (): Spend => ({ iterations: 0, tokens: { input: 0, output: 0 }, usd: 0, ms: 0 }) +const noTrace = { + status: 'unavailable', + reason: 'executor-did-not-expose-trace-source', +} as const satisfies WorkerTraceEvidence + +const toolSpan = { + spanId: 'worker-trace-t0', + runId: 'worker-trace', + kind: 'tool', + name: 'read_file', + toolName: 'read_file', + args: { path: 'actual.ts' }, + result: { text: 'structured tool result' }, + status: 'ok', + startedAt: 100, + endedAt: 105, +} as const satisfies ToolSpan +const traceArtifact = { + schemaVersion: WORKER_TOOL_TRACE_SCHEMA_VERSION, + spans: [toolSpan], +} +const traceRef = contentAddress(traceArtifact) +const availableTrace = { + status: 'available', + traceRef, + spanCount: 1, +} as const satisfies WorkerTraceEvidence + +function traceBlobStore(outputRef: string, output: unknown): ResultBlobStore { + return { + get: async (ref) => { + if (ref === traceRef) return traceArtifact + if (ref === outputRef) return output + return undefined + }, + put: async () => {}, + } +} + +async function traceSummary(trace: TraceAnalysisStore) { + const overview = await trace.getOverview() + return { + traces: overview.total_traces, + tools: overview.tool_names, + traceIds: overview.sample_trace_ids, + } +} + function mockScope() { const sent: Array<{ id: string; msg: unknown }> = [] const spawns: Array<{ task: unknown; opts: { budget: unknown; label: string } }> = [] @@ -30,6 +84,7 @@ function mockScope() { budget: { maxIterations: 1, maxTokens: 10 }, spent: zeroSpend(), outRef: 'blob:w1', + trace: noTrace, }, ] let admit = true @@ -242,6 +297,7 @@ describe('coordination tools', () => { outRef: 'blob:a', verdict: { score: 1, valid: true }, spent: zeroSpend(), + trace: noTrace, seq: 0, } let deliveredKey: string | undefined @@ -321,6 +377,7 @@ describe('coordination tools', () => { valid: true, outRef: 'blob:a', spent: zeroSpend(), + trace: noTrace, live: 1, freeSlots: 0, }) @@ -443,6 +500,7 @@ describe('coordination tools', () => { outRef: 'blob:w7', verdict: { valid: true, score: 0.83 }, spent: zeroSpend(), + trace: noTrace, seq: 0, }, ] @@ -466,6 +524,7 @@ describe('coordination tools', () => { valid: true, outRef: 'blob:w7', spent: zeroSpend(), + trace: noTrace, freeSlots: null, }) expect(await tool(tb, 'await_event').handler({ kinds: ['settled'] })).toEqual({ @@ -524,6 +583,7 @@ describe('coordination tools', () => { outRef: 'blob:w0', verdict: { valid: true, score: 0.5 }, spent: zeroSpend(), + trace: noTrace, seq: 0, } release() @@ -657,11 +717,11 @@ describe('coordination tools', () => { it('list_analysts surfaces the menu and run_analyst applies a lens to a settled worker', async () => { const { scope } = mockScope() - const traceBlobs: ResultBlobStore = { - get: async (ref) => (ref === 'blob:w1' ? { messages: ['trace'] } : undefined), - put: async () => {}, - } - const seen: Array<{ kind: string; trace: unknown }> = [] + Object.assign(scope.view.nodes.find((node) => node.id === 'w1')!, { trace: availableTrace }) + const traceBlobs = traceBlobStore('blob:w1', { + messages: ['WORKER PROSE THAT MUST NEVER REACH A TRACE ANALYST'], + }) + const seen: Array<{ kind: string; trace: Awaited> }> = [] const tb = createCoordinationTools({ scope, blobs: traceBlobs, @@ -670,7 +730,7 @@ describe('coordination tools', () => { analysts: { kinds: [{ id: 'completeness', description: 'unfinished work', area: 'failure-mode' }], run: async (kind, trace) => { - seen.push({ kind, trace }) + seen.push({ kind, trace: await traceSummary(trace) }) return [{ claim: 'X missing' }] }, }, @@ -683,7 +743,12 @@ describe('coordination tools', () => { findings: [{ claim: 'X missing' }], }, ) - expect(seen).toEqual([{ kind: 'completeness', trace: { messages: ['trace'] } }]) + expect(seen).toEqual([ + { + kind: 'completeness', + trace: { traces: 1, tools: ['read_file'], traceIds: ['worker-trace'] }, + }, + ]) expect(await tool(tb, 'run_analyst').handler({ kind: 'completeness', workerId: 'w0' })).toEqual( { error: expect.stringContaining('has not settled'), @@ -691,6 +756,34 @@ describe('coordination tools', () => { ) }) + it('refuses trace analysis when a settled worker has no structured tool spans', async () => { + const { scope } = mockScope() + let analystCalls = 0 + const tb = createCoordinationTools({ + scope, + blobs: traceBlobStore('blob:w1', { + messages: ['plausible-looking worker prose is still not tool evidence'], + }), + makeWorkerAgent, + perWorker: { maxIterations: 1, maxTokens: 10 }, + analysts: { + kinds: [{ id: 'completeness', description: 'unfinished work', area: 'failure-mode' }], + run: async () => { + analystCalls += 1 + return [] + }, + }, + }) + + await expect( + tool(tb, 'run_analyst').handler({ kind: 'completeness', workerId: 'w1' }), + ).resolves.toEqual({ + error: expect.stringContaining('trace evidence is missing'), + trace: noTrace, + }) + expect(analystCalls).toBe(0) + }) + it('await_event bumps a blocking question ahead of a non-blocking one (urgency→priority)', async () => { const { scope } = mockScope() const tb = createCoordinationTools({ @@ -903,6 +996,7 @@ describe('coordination tools', () => { outRef: 'blob:w7', verdict: { valid: false, score: 0.1 }, spent: zeroSpend(), + trace: availableTrace, seq: 0, }, ] @@ -913,15 +1007,21 @@ describe('coordination tools', () => { const emitted: string[] = [] const tb = createCoordinationTools({ scope: drainScope, - blobs: { - get: async (ref) => (ref === 'blob:w7' ? { messages: ['trace'] } : undefined), - put: async () => {}, - }, + blobs: traceBlobStore('blob:w7', { + messages: ['worker final prose must not be analyzed as a trace'], + }), makeWorkerAgent, perWorker: { maxIterations: 1, maxTokens: 10 }, analysts: { kinds: [{ id: 'completeness', description: 'unfinished work', area: 'failure-mode' }], - run: async () => [{ claim: 'stub left in place' }], + run: async (_kind, trace) => { + expect(await traceSummary(trace)).toEqual({ + traces: 1, + tools: ['read_file'], + traceIds: ['worker-trace'], + }) + return [{ claim: 'stub left in place' }] + }, }, analyzeOnSettle: ['completeness'], onEvent: (e) => emitted.push(e.type), @@ -936,6 +1036,7 @@ describe('coordination tools', () => { valid: false, outRef: 'blob:w7', spent: zeroSpend(), + trace: availableTrace, freeSlots: null, }) // The analyze-on-settle finding is now queued; the next pull surfaces it. @@ -962,6 +1063,7 @@ describe('coordination tools', () => { outRef: 'blob:w-retry', verdict: { valid: true, score: 0.8 }, spent: zeroSpend(), + trace: noTrace, seq: 0, }, ] @@ -1009,6 +1111,7 @@ describe('coordination tools', () => { outRef: 'blob:w8', verdict: { valid: true, score: 1 }, spent: zeroSpend(), + trace: noTrace, seq: 0, }, ] @@ -1023,6 +1126,7 @@ describe('coordination tools', () => { type: 'settled', settled: 'w8', valid: true, + trace: noTrace, }) expect(await tool(tb, 'await_event').handler({ kinds: ['settled'] })).toEqual({ idle: true, @@ -1040,6 +1144,7 @@ describe('coordination tools', () => { outRef: 'blob:w9', verdict: { valid: true, score: 1 }, spent: zeroSpend(), + trace: noTrace, seq: 0, }, ] @@ -1058,6 +1163,7 @@ describe('coordination tools', () => { // The drained settled event was queued, not lost — a caller that asks for it still gets it. expect(await tool(tb, 'await_event').handler({ kinds: ['settled'] })).toMatchObject({ settled: 'w9', + trace: noTrace, }) }) diff --git a/tests/runtime/worker-trace-evidence.test.ts b/tests/runtime/worker-trace-evidence.test.ts new file mode 100644 index 00000000..a3738da9 --- /dev/null +++ b/tests/runtime/worker-trace-evidence.test.ts @@ -0,0 +1,220 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + FileResultBlobStore, + FileSpawnJournal, + InMemoryResultBlobStore, + InMemorySpawnJournal, + replaySpawnTree, +} from '../../src/durable/spawn-journal' +import { createBudgetPool } from '../../src/runtime/supervise/budget' +import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { createScope } from '../../src/runtime/supervise/scope' +import { workerTraceAnalysisStore } from '../../src/runtime/supervise/trace-evidence' +import { createPushTraceSource } from '../../src/runtime/supervise/trace-source' +import type { + Agent, + AgentSpec, + Executor, + ExecutorResult, + ResultBlobStore, + SpawnJournal, +} from '../../src/runtime/supervise/types' + +const spent = { + iterations: 1, + tokens: { input: 2, output: 3 }, + usd: 0.01, + ms: 4, +} + +function makeAgent( + executor: Executor, +): Agent & { executorSpec: AgentSpec } { + return { + name: 'trace worker', + act: async () => undefined, + executorSpec: { profile: { name: 'trace worker' }, harness: null, executor }, + } +} + +function makeScope(root: string, journal: SpawnJournal, blobs: ResultBlobStore) { + return createScope({ + parentId: root, + root, + pool: createBudgetPool({ maxIterations: 2, maxTokens: 100 }, () => 0), + journal, + blobs, + executors: createExecutorRegistry(), + seams: {}, + depth: 0, + maxDepth: 4, + signal: new AbortController().signal, + now: () => 100, + }) +} + +describe('durable worker trace evidence', () => { + it('persists exact tool spans and the result blob before journaling a live settlement', async () => { + const root = 'live-trace' + const innerJournal = new InMemorySpawnJournal() + const blobs = new InMemoryResultBlobStore() + let checkedTerminalReferences = false + const journal: SpawnJournal = { + loadTree: (id) => innerJournal.loadTree(id), + beginTree: (id, at) => innerJournal.beginTree(id, at), + async appendEvent(id, event) { + if (event.kind === 'settled') { + expect(event.trace?.status).toBe('available') + if (event.trace?.status === 'available') { + expect(await blobs.get(event.trace.traceRef)).toBeDefined() + } + if (event.status === 'done') expect(await blobs.get(event.outRef!)).toBeDefined() + checkedTerminalReferences = true + } + await innerJournal.appendEvent(id, event) + }, + } + await journal.beginTree(root, new Date(100).toISOString()) + + const trace = createPushTraceSource({ runId: 'live-worker', now: () => 101 }) + const output = { final: 'worker prose is a result, not a trace' } + const executor: Executor = { + runtime: 'router-tools', + async execute(): Promise> { + trace.record({ + toolName: 'read_file', + args: { path: 'src/index.ts' }, + result: { bytes: 42 }, + }) + return { outRef: 'executor-local-ref', out: output, spent } + }, + traceSource: () => trace.source, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => ({ outRef: 'executor-local-ref', out: output, spent }), + } + const scope = makeScope(root, journal, blobs) + expect( + scope.spawn(makeAgent(executor), 'inspect', { + budget: { maxIterations: 1, maxTokens: 50 }, + label: 'live trace', + }).ok, + ).toBe(true) + + const settled = await scope.next() + expect(settled).toMatchObject({ + kind: 'done', + out: output, + trace: { status: 'available', spanCount: 1 }, + }) + expect(checkedTerminalReferences).toBe(true) + if (settled?.trace.status !== 'available') throw new Error('expected trace evidence') + const store = await workerTraceAnalysisStore(settled.trace, blobs) + await expect(store.getOverview()).resolves.toMatchObject({ + total_traces: 1, + sample_trace_ids: ['live-worker'], + tool_names: ['read_file'], + }) + }) + + it('marks an empty Codex bridge source unavailable and never treats final prose as evidence', async () => { + const root = 'codex-no-trace' + const journal = new InMemorySpawnJournal() + const blobs = new InMemoryResultBlobStore() + await journal.beginTree(root, new Date(100).toISOString()) + const trace = createPushTraceSource({ runId: 'codex-worker' }) + const output = { final: 'I used many tools, trust me' } + const executor: Executor = { + runtime: 'codex', + execute: async () => ({ outRef: 'executor-local-ref', out: output, spent }), + traceSource: () => trace.source, + teardown: async () => ({ destroyed: true }), + resultArtifact: () => ({ outRef: 'executor-local-ref', out: output, spent }), + } + const scope = makeScope(root, journal, blobs) + expect( + scope.spawn(makeAgent(executor), 'inspect', { + budget: { maxIterations: 1, maxTokens: 50 }, + label: 'codex no trace', + }).ok, + ).toBe(true) + + const settled = await scope.next() + expect(settled).toMatchObject({ + kind: 'done', + out: output, + trace: { status: 'unavailable', reason: 'no-tool-spans-captured' }, + }) + if (!settled) throw new Error('expected settlement') + await expect(workerTraceAnalysisStore(settled.trace, blobs)).rejects.toThrow( + 'trace evidence is missing', + ) + }) + + it('retains partial structured trace evidence across a worker crash and process restart', async () => { + const dir = await mkdtemp(join(tmpdir(), 'worker-trace-restart-')) + try { + const root = 'crash-restart-trace' + const journalPath = join(dir, 'spawn.jsonl') + const blobPath = join(dir, 'blobs') + const journal = new FileSpawnJournal(journalPath) + const blobs = new FileResultBlobStore(blobPath) + await journal.beginTree(root, new Date(100).toISOString()) + const trace = createPushTraceSource({ runId: 'crashed-worker', now: () => 102 }) + const executor: Executor = { + runtime: 'router-tools', + async execute() { + trace.record({ + toolName: 'write_file', + args: { path: 'partial.ts' }, + status: 'error', + result: { error: 'disk full' }, + }) + throw new Error('provider session crashed') + }, + traceSource: () => trace.source, + teardown: async () => ({ destroyed: true }), + resultArtifact() { + throw new Error('result unavailable after crash') + }, + } + const scope = makeScope(root, journal, blobs) + expect( + scope.spawn(makeAgent(executor), 'mutate', { + budget: { maxIterations: 1, maxTokens: 50 }, + label: 'crashed trace', + }).ok, + ).toBe(true) + await expect(scope.next()).resolves.toMatchObject({ + kind: 'down', + reason: 'provider session crashed', + trace: { status: 'available', spanCount: 1 }, + }) + + const restartedJournal = new FileSpawnJournal(journalPath) + const restartedBlobs = new FileResultBlobStore(blobPath) + const replayed = await replaySpawnTree(restartedJournal, restartedBlobs, root) + expect(replayed).toHaveLength(1) + expect(replayed[0]).toMatchObject({ + kind: 'down', + reason: 'provider session crashed', + trace: { status: 'available', spanCount: 1 }, + }) + const resumed = replayed[0] + if (resumed?.trace.status !== 'available') { + throw new Error('expected replayed trace evidence') + } + const store = await workerTraceAnalysisStore(resumed.trace, restartedBlobs) + await expect(store.getOverview()).resolves.toMatchObject({ + total_traces: 1, + sample_trace_ids: ['crashed-worker'], + tool_names: ['write_file'], + errors: { trace_count: 1, span_count: 1 }, + }) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) +}) From 2cee3ae4328d5f1e77e65eb0d35e5293bc6c4d04 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 06:13:30 -0600 Subject: [PATCH 06/12] fix(supervise): complete profile-aware execution paths --- docs/api/primitive-catalog.md | 35 +- src/agent/index.ts | 7 + src/agent/profile-materialization.ts | 206 ++++++++- src/mcp/worktree-harness.ts | 28 +- src/runtime/router-client.ts | 2 +- src/runtime/strategy.ts | 2 +- .../supervise/worktree-cli-executor.ts | 7 +- src/runtime/supervise/worktree-fanout.ts | 62 ++- tests/candidate-execution-execute.test.ts | 2 +- .../nested-coordination-durability.test.ts | 139 ++++++ tests/kernel/strategy-evolution.test.ts | 35 +- tests/kernel/strategy-suite.test.ts | 10 +- .../supervise-global-concurrency.test.ts | 399 ++++++++++++++++++ tests/kernel/worktree-loop.test.ts | 38 +- tests/mcp/worktree-harness.test.ts | 63 +-- tests/profile-materialization.test.ts | 172 ++++++++ tests/runtime/worktree-cli-executor.test.ts | 2 +- 17 files changed, 1090 insertions(+), 119 deletions(-) create mode 100644 tests/kernel/nested-coordination-durability.test.ts create mode 100644 tests/kernel/supervise-global-concurrency.test.ts diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 85991181..5c40edd1 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.109.1` and `@tangle-network/agent-eval@0.135.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.109.1` and `@tangle-network/agent-eval@0.135.3` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,10 +15,11 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 401 exports. +Import from `@tangle-network/agent-runtime` — 404 exports. | Symbol | Kind | Summary | |---|---|---| +| `agentCandidateProfileAsAgentProfile` | function | Convert the candidate profile contract into the portable interface profile it represents. | | `agenticGenerator` | function | Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. | | `applyExactAgentProfileDiff` | function | Apply one exact diff and reject any value that cannot be preserved canonically. | | `applyRolloutPolicyToProfile` | function | Persist a detached policy under the profile extension without mutating the input. | @@ -248,6 +249,8 @@ Import from `@tangle-network/agent-runtime` — 401 exports. | `SupervisorFinalizer` | type | The finalization seam: ledger in, output (or `undefined` = nothing deliverable) out. | | `VerifiedAgentCandidateTaskOutcome` | type | Branded task outcome that has survived independent evaluator verification. | | `Verifier` | type | Verifies the edited worktree. Sync or async; throws only on a setup fault | +| `WorkerTraceEvidence` | type | Durable proof of a worker's structured tool trace, or the exact reason it is unavailable. | +| `WorkerTraceUnavailableReason` | type | Why Runtime cannot provide structured tool-call evidence for one settled execution. | | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | **Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `DriverLoopGeneratorOptions`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeOptions`, `ImproveCodeResult`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImprovementCandidate`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. @@ -499,7 +502,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 624 exports. +Import from `@tangle-network/agent-runtime/kernel` — 631 exports. | Symbol | Kind | Summary | |---|---|---| @@ -522,13 +525,14 @@ Import from `@tangle-network/agent-runtime/kernel` — 624 exports. | `breadthStrategy` | function | BREADTH: K independent rollouts (each own artifact), verifier picks the best. | | `buildSteerContext` | function | Build the `SteerContext` a combinator reads to steer (its `loopUntil.until`, `widen` gate, any | | `canDisplace` | function | The repair keep-best guard: a challenger displaces the incumbent only when it is | +| `captureWorkerTraceEvidence` | function | Collect and persist one executor's structured tool trace without changing its task outcome. | | `collectAgentTurn` | function | Drain a `streamAgentTurn` stream (or any `RuntimeStreamEvent` stream that | | `compareCheckOutcomes` | function | The selection order: crash < ran; then official pass-fraction; authored guesses only | | `completionAuthorizes` | function | Decide whether a `CompletionVerdict` may end the node under the policy: authority scales with the verdict's determinism, and probabilistic verdicts must clear `minConfidence`. | | `composeCheckSources` | function | Concatenate check sources (official first by convention — ordering does not affect | | `computeFindingId` | function | Compute the stable finding_id from the identity-defining fields. | | `connectStdioMcp` | function | Spawn a trusted host command, complete the stdio MCP handshake, and return | -| `contentAddress` | function | Mint the content-addressed `outRef` for a result artifact: `sha256:` over a | +| `contentAddress` | function | Stable content address shared by result and trace artifacts. | | `createActivityLog` | function | Create a bounded activity ring. `limit` caps memory for a worker that runs thousands of tools. | | `createAgentEnvironmentProviderRegistry` | function | Create a registry that resolves provider names to concrete provider instances. | | `createBudgetPool` | function | Create a conserved reservation pool from a root `Budget`. `now()` is injected so the | @@ -602,6 +606,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 624 exports. | `openSandboxRun` | function | Open a sandbox run. Harness-agnostic: the harness lives in | | `pairwiseSignificance` | function | Compare EVERY profile pair on the scenarios they both ran — paired-bootstrap effect + CI, a real | | `panel` | function | `panel(spec)` — spawn the M judge children over the SAME artifact, drain their settlements, | +| `parseWorkerToolTraceArtifact` | function | Validate a stored trace artifact before an analyst or replay trusts it. | | `patchDelivered` | function | Build the `DeliverableSpec`: `check(artifact)` runs the shared mechanical | | `pendingWaits` | function | The waits a journaled tree shows as ARMED but never woken — what a resumed run re-arms with the | | `pickBestDelivered` | function | The single argmax both the default finalizer and `finalizeBestDelivered` share: highest | @@ -675,6 +680,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 624 exports. | `watchTrace` | function | Subscribe to a `TraceSource` and run the streaming detectors over its live spans. Returns an | | `widen` | function | `widen(spec)` — the streaming spawn-on-completion driver. Spawns the seed lineages, then REACTS | | `workerFromBackend` | function | Build the worker seam from a backend (WHERE workers run) + an optional completion oracle (the | +| `workerTraceAnalysisStore` | function | Rehydrate exact persisted spans through agent-eval's one bounded trace-analysis adapter. | | `worktreeFanout` | function | Build the worktree fanout combinator. Run it with `runPersonified({ persona, shape, task, budget })` | | `adaptiveRefine` | const | A NEW strategy, authored from the steps (~20 lines): refine, but when a steered shot | | `assertTraceDerivedFindings` | const | Reject analyst findings derived from evaluation scores instead of execution traces. | @@ -700,6 +706,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 624 exports. | `sample` | const | Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N / resample). | | `sampleThenRefine` | const | The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept open), | | `strategyAuthorContract` | const | The compressed consumable a skill carries: everything an author needs to emit a loop. | +| `WORKER_TOOL_TRACE_SCHEMA_VERSION` | const | Schema version for content-addressed worker tool-trace artifacts. | | `FileCoordinationLog` | class | FS-backed `CoordinationLog`: append-only JSONL, fsynced per record. | | `FileCorpus` | class | JSONL on disk — one validated `CorpusRecord` per line, append-only. `query` replays the whole | | `FileResultBlobStore` | class | FS `ResultBlobStore`. One JSON file per artifact under `dir`, named by a | @@ -899,6 +906,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 624 exports. | `WidenSpec` | interface | `widen({ gate })` (G5) — the STREAMING spawn-on-completion driver. Unlike the static-fanout | | `WorkerProgress` | interface | The full live view of one worker, as `observe_agent` returns it mid-flight. | | `WorkerSpawnContext` | interface | Immutable task, allocation, identity attribution, and semantic key supplied while a manager's | +| `WorkerToolTraceArtifact` | interface | Bytes stored under `WorkerTraceEvidence.traceRef`. | | `WorktreeCommandResult` | interface | Outcome of one verification command run in the worktree (test or typecheck). | | `WorktreeHarnessResult` | interface | The canonical result of one worktree-harness run, projected by each port to its own shape. | | `WorktreeProfileMaterializationReceipt` | interface | Proof of the profile inputs delivered before the worker process started. | @@ -981,6 +989,8 @@ Import from `@tangle-network/agent-runtime/kernel` — 624 exports. | `Widen` | type | `widen(spec)` — build the streaming progressive-widening combinator. | | `WidenDecision` | type | A widening decision: extend one lineage by one child, or stop widening. `flatWidenGate` | | `WinnerStrategy` | type | Built-in valid-only winner strategies for `selectValidWinner` (selector≠judge): best gated-valid | +| `WorkerTraceEvidence` | type | Durable proof of a worker's structured tool trace, or the exact reason it is unavailable. | +| `WorkerTraceUnavailableReason` | type | Why Runtime cannot provide structured tool-call evidence for one settled execution. | | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | | `WorktreePatchArtifact` | type | Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured | @@ -1125,10 +1135,11 @@ Import from `@tangle-network/agent-runtime/primeintellect` — 30 exports. ### Candidate execution — immutable prepare, run, grade, and receipt -Import from `@tangle-network/agent-runtime/candidate-execution` — 104 exports. +Import from `@tangle-network/agent-runtime/candidate-execution` — 105 exports. | Symbol | Kind | Summary | |---|---|---| +| `agentCandidateProfileAsAgentProfile` | function | Convert the candidate profile contract into the portable interface profile it represents. | | `applyExactAgentProfileDiff` | function | Apply one exact diff and reject any value that cannot be preserved canonically. | | `assertCandidateProfileBinding` | function | Prove the measured generic profile and sealed candidate profile describe the same behavior. | | `buildAgentCandidateBundle` | function | Compile one measured profile/code candidate into the immutable execution | @@ -1414,7 +1425,7 @@ Import from `@tangle-network/agent-eval` — 10 exports. ### STATISTICS — significance, intervals, effect size -Import from `@tangle-network/agent-eval` — 53 exports. +Import from `@tangle-network/agent-eval` — 61 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1433,12 +1444,17 @@ Import from `@tangle-network/agent-eval` — 53 exports. | `mcnemarPower` | function | Power of a McNemar test at a given number of paired observations, the inverse | | `mcnemarRequiredN` | function | Number of paired observations needed for a McNemar test to reach a target | | `mulberry32` | function | Tiny seedable PRNG (mulberry32) — deterministic resampling/shuffling, not | +| `pairedBinaryScale` | function | The common positive level `s` such that EVERY value across both paired arms is | | `pairedBootstrap` | function | Paired bootstrap on (after − before) deltas. Returns a CI on the chosen | | `pairedCohensDz` | function | Cohen's dz for paired observations: mean(after - before) divided by the | +| `pairedDecisionShape` | function | Which estimator {@link decidePairedPromotion} would use on this data, and the | | `pairedDeltaTest` | function | Tests whether a paired candidate-minus-baseline delta clears a threshold. | +| `pairedDeltaTieFraction` | function | Fraction of paired observations whose delta is an exact tie (\|after − before\| | | `pairedEvalueSequence` | function | Run the paired e-value sequence over an in-order delta stream. | | `pairedMde` | function | Minimum detectable paired effect (standardised units) for a target paired | | `pairedRiskDifference` | function | Paired risk difference (the effect-size companion to {@link mcnemar}): the | +| `pairedRiskDifferenceExact` | function | Paired risk difference with the EXACT CONDITIONAL interval — the estimator a | +| `pairedRiskDifferenceScore` | function | Paired risk difference with TANGO'S (1998) SCORE INTERVAL — the estimator a | | `pairedSignTest` | function | Exact one-sided sign test over paired differences. | | `pairedTTest` | function | Paired t-test — before/after measurements on the SAME items. | | `partialCredit` | function | Partial credit: returns 0-1 ratio of current toward target | @@ -1452,9 +1468,12 @@ Import from `@tangle-network/agent-eval` — 53 exports. | `wilcoxonSignedRank` | function | Wilcoxon signed-rank — paired, no distributional assumption on the deltas. | | `wilson` | function | Wilson score interval for a binomial proportion. Correct at small n and near | | `normalizeScores` | const | Identity: dimensions already follow "higher = better" by prompt convention | +| `ExactRiskDifferenceResult` | interface | A paired binary effect size with an EXACT interval and the exact test that | | `McNemarResult` | interface | Result of a McNemar paired-binary significance test. | +| `PairedMcNemarEvidence` | interface | McNemar's exact paired-binary evidence, on the two-point path only. | | `ProportionInterval` | interface | A binomial proportion estimate with a confidence interval. | | `RiskDifferenceResult` | interface | A paired binary effect size (treatment rate − control rate) with a CI. | +| `ScoreRiskDifferenceResult` | interface | A paired binary effect size with an interval that is valid at a NONZERO | **Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BootstrapOptions`, `BootstrapResult`, `ClusterBootstrapInterval`, `CorpusAgreementOptions`, `CorpusAgreementPerDimension`, `CorpusAgreementReport`, `CorpusScoreRecord`, `EProcess`, `EProcessOptions`, `EProcessState`, `EProcessStep`, `PairedBootstrapOptions`, `PairedBootstrapResult`, `WeightedCompositeInput`, `WeightedCompositeResult`, `CliffsMagnitude`. @@ -1499,8 +1518,8 @@ Import from `@tangle-network/agent-eval/campaign` — 329 exports. | `fsCampaignStorage` | function | Node-filesystem storage — the default. Lazily requires `node:fs` so the | | `gepaOptimizationMethod` | function | Turn an optional GEPA installation into an `OptimizationMethod`. | | `gitWorktreeAdapter` | function | Git-backed `WorktreeAdapter`: creates isolated worktrees on fresh branches, commits agent changes, and discards losers. | -| `heldOutGate` | function | Composable held-out gate: ships only when the PAIRED bootstrap CI lower bound | -| `heldoutSignificance` | function | Significance of the held-out composite lift: ship only when the paired | +| `heldOutGate` | function | Composable held-out gate: ships only when the lower bound of the DECIDING | +| `heldoutSignificance` | function | Significance of the held-out composite lift: ship only when the lower bound | | `inMemoryCampaignStorage` | function | In-memory storage for filesystem-less runtimes. Artifacts + trace spans | | `isProposedCandidate` | function | Type guard: a proposal carrying its rationale vs a bare | | `isTransientTransportFailure` | function | True when the error text describes an infrastructure hiccup that should be | diff --git a/src/agent/index.ts b/src/agent/index.ts index 6ee24759..1c3fdd63 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -34,6 +34,7 @@ export { createSurfaceImprovementProposer } from './improvement-adapter' export type { AgentProfileMaterializationAxis, AssertProfileMaterializationOptions, + CanonicalAgentProfileMaterializationAxis, DefineProfileMaterializationContractOptions, KnownAgentProfileMaterializationAxis, ProfileMaterializationContract, @@ -43,12 +44,18 @@ export type { export { AGENT_PROFILE_MATERIALIZATION_AXES, assertProfileMaterialization, + controlProfileMaterialization, defineProfileMaterializationContract, + fullProfileMaterialization, + profileMaterializationAxes, + promptControlProfileMaterialization, + promptModelProfileMaterialization, promptOnlyProfileMaterialization, promptResourceProfileMaterialization, renderProfileMaterializationIssues, sandboxActProfileMaterialization, validateProfileMaterialization, + worktreeCliProfileMaterialization, } from './profile-materialization' export type { CreateSandboxActOptions, SandboxActComposeOverrides } from './sandbox-act' export { createSandboxAct } from './sandbox-act' diff --git a/src/agent/profile-materialization.ts b/src/agent/profile-materialization.ts index 3411bcd3..888512ae 100644 --- a/src/agent/profile-materialization.ts +++ b/src/agent/profile-materialization.ts @@ -1,10 +1,20 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' import { ValidationError } from '../errors' /** Known AgentProfile axes a run path may or may not carry into execution. */ export const AGENT_PROFILE_MATERIALIZATION_AXES = [ 'identity', 'name', + 'description', + 'version', + 'tags', 'model', + 'modelDefault', + 'modelSmall', + 'modelProvider', + 'modelReasoningEffort', + 'modelMetadata', + 'harness', 'prompt', 'systemPrompt', 'instructions', @@ -15,6 +25,7 @@ export const AGENT_PROFILE_MATERIALIZATION_AXES = [ 'resourceTools', 'resourceAgents', 'commands', + 'resourceFailOnError', 'tools', 'permissions', 'mcp', @@ -36,6 +47,15 @@ export type AgentProfileMaterializationAxis = | KnownAgentProfileMaterializationAxis | `custom:${string}` +type AgentProfileIdentityProperty = 'name' | 'description' | 'version' | 'tags' +type AgentProfilePropertyMaterializationAxis = Exclude< + keyof AgentProfile, + AgentProfileIdentityProperty +> + +/** Canonical AgentProfile axes used when checking one complete profile. */ +export type CanonicalAgentProfileMaterializationAxis = KnownAgentProfileMaterializationAxis + /** Declares which AgentProfile axes a concrete run path really carries. */ export interface ProfileMaterializationContract { /** Human-readable run path, e.g. `createSandboxAct` or `prompt-only-message`. */ @@ -76,6 +96,14 @@ const AXIS_PARENTS: Partial< Record > = { name: 'identity', + description: 'identity', + version: 'identity', + tags: 'identity', + modelDefault: 'model', + modelSmall: 'model', + modelProvider: 'model', + modelReasoningEffort: 'model', + modelMetadata: 'model', systemPrompt: 'prompt', instructions: 'prompt', files: 'resources', @@ -84,30 +112,110 @@ const AXIS_PARENTS: Partial< resourceTools: 'resources', resourceAgents: 'resources', commands: 'resources', + resourceFailOnError: 'resources', mcpConnections: 'mcp', } -/** Materialization contract for `createSandboxAct`, which forwards the full AgentProfile. */ -export const sandboxActProfileMaterialization = defineProfileMaterializationContract({ - name: 'createSandboxAct', +const canonicalAgentProfilePropertyAxes = [ + 'prompt', + 'model', + 'harness', + 'permissions', + 'tools', + 'mcp', + 'connections', + 'subagents', + 'resources', + 'hooks', + 'modes', + 'confidential', + 'metadata', + 'extensions', +] as const satisfies readonly AgentProfilePropertyMaterializationAxis[] + +type MissingAgentProfilePropertyMaterializationAxis = Exclude< + AgentProfilePropertyMaterializationAxis, + (typeof canonicalAgentProfilePropertyAxes)[number] +> +const agentProfilePropertyAxesAreExhaustive: MissingAgentProfilePropertyMaterializationAxis extends never + ? true + : never = true +void agentProfilePropertyAxesAreExhaustive + +const fullProfileMaterializationAxes = [ + 'identity', + ...canonicalAgentProfilePropertyAxes, +] as const satisfies readonly CanonicalAgentProfileMaterializationAxis[] + +/** Materialization contract for a run path that executes every canonical AgentProfile axis. */ +export const fullProfileMaterialization = defineProfileMaterializationContract({ + name: 'full-profile-execution', + axes: fullProfileMaterializationAxes, +}) + +/** + * Materialization contract for an intentionally limited prompt-and-model execution path. + * Identity, harness, and metadata are control fields consumed for naming, placement, + * authorization, and durable attribution; they are carried without adding worker behavior. + * Every behavioral axis other than prompt and model remains unsupported. + */ +export const promptModelProfileMaterialization = defineProfileMaterializationContract({ + name: 'prompt-model-execution', + axes: ['name', 'systemPrompt', 'instructions', 'modelDefault', 'harness', 'metadata'], +}) + +/** + * Materialization contract for a local coding CLI in an isolated git worktree. + * The shared workspace materializer carries native tools, permissions, MCP, hooks, subagents, + * modes, and file-backed resources when the selected CLI supports their exact values. Runtime + * placement concerns (hub connections and confidential execution), provider-native extensions, + * unused model hints, and `resources.failOnError` are deliberately absent so they fail before a + * worktree or executor is created rather than being mistaken for an effective candidate change. + */ +export const worktreeCliProfileMaterialization = defineProfileMaterializationContract({ + name: 'worktree-cli-execution', axes: [ - 'identity', - 'model', - 'prompt', - 'resources', - 'tools', + 'name', + 'systemPrompt', + 'instructions', + 'modelDefault', + 'modelReasoningEffort', + 'harness', 'permissions', + 'tools', 'mcp', - 'connections', 'subagents', + 'files', + 'resourceTools', + 'skills', + 'resourceAgents', + 'commands', + 'resourceInstructions', 'hooks', 'modes', - 'confidential', 'metadata', - 'extensions', ], }) +/** Materialization contract for a raw process path that carries only control/identity fields. */ +export const controlProfileMaterialization = defineProfileMaterializationContract({ + name: 'control-only-execution', + axes: ['name', 'harness', 'metadata'], +}) + +/** Materialization contract for an injected inference function whose surrounding driver still + * applies the profile prompt, name, placement, and metadata, but not model selection. */ +export const promptControlProfileMaterialization = defineProfileMaterializationContract({ + name: 'prompt-control-execution', + axes: ['name', 'systemPrompt', 'instructions', 'harness', 'metadata'], +}) + +/** Materialization contract for `createSandboxAct`, which forwards the full AgentProfile. */ +export const sandboxActProfileMaterialization = defineProfileMaterializationContract({ + name: 'createSandboxAct', + axes: fullProfileMaterialization.axes, +}) + /** Materialization contract for a run path that only injects prompt text. */ export const promptOnlyProfileMaterialization = defineProfileMaterializationContract({ name: 'prompt-only-message', @@ -134,6 +242,57 @@ export function defineProfileMaterializationContract( } } +/** + * Return the exact canonical axes a complete profile actually requests. Compound prompt, model, + * identity, and resource objects are split so a path cannot claim an entire object while silently + * dropping one of its fields. + * Empty strings, arrays, and nested records do not claim support; explicit + * scalar values such as `false` and `0` remain meaningful requests. + */ +export function profileMaterializationAxes( + profile: AgentProfile, +): readonly CanonicalAgentProfileMaterializationAxis[] { + const axes: CanonicalAgentProfileMaterializationAxis[] = [] + addIfRequested(axes, 'name', profile.name) + addIfRequested(axes, 'description', profile.description) + addIfRequested(axes, 'version', profile.version) + addIfRequested(axes, 'tags', profile.tags) + addIfRequested(axes, 'systemPrompt', profile.prompt?.systemPrompt) + addIfRequested(axes, 'instructions', profile.prompt?.instructions) + addIfRequested(axes, 'modelDefault', profile.model?.default) + addIfRequested(axes, 'modelSmall', profile.model?.small) + addIfRequested(axes, 'modelProvider', profile.model?.provider) + addIfRequested(axes, 'modelReasoningEffort', profile.model?.reasoningEffort) + addIfRequested(axes, 'modelMetadata', profile.model?.metadata) + addIfRequested(axes, 'harness', profile.harness) + addIfRequested(axes, 'permissions', profile.permissions) + addIfRequested(axes, 'tools', profile.tools) + addIfRequested(axes, 'mcp', profile.mcp) + addIfRequested(axes, 'connections', profile.connections) + addIfRequested(axes, 'subagents', profile.subagents) + addIfRequested(axes, 'files', profile.resources?.files) + addIfRequested(axes, 'resourceTools', profile.resources?.tools) + addIfRequested(axes, 'skills', profile.resources?.skills) + addIfRequested(axes, 'resourceAgents', profile.resources?.agents) + addIfRequested(axes, 'commands', profile.resources?.commands) + addIfRequested(axes, 'resourceInstructions', profile.resources?.instructions) + addIfRequested(axes, 'resourceFailOnError', profile.resources?.failOnError) + addIfRequested(axes, 'hooks', profile.hooks) + addIfRequested(axes, 'modes', profile.modes) + addIfRequested(axes, 'confidential', profile.confidential) + addIfRequested(axes, 'metadata', profile.metadata) + addIfRequested(axes, 'extensions', profile.extensions) + return axes +} + +function addIfRequested( + axes: CanonicalAgentProfileMaterializationAxis[], + axis: CanonicalAgentProfileMaterializationAxis, + value: unknown, +): void { + if (hasNonEmptyMaterializationValue(value)) axes.push(axis) +} + /** Return every changed profile axis that the selected run path would drop. */ export function validateProfileMaterialization( options: ValidateProfileMaterializationOptions, @@ -225,3 +384,28 @@ function isAxisSupported( } return false } + +function hasNonEmptyMaterializationValue(root: unknown): boolean { + const pending: unknown[] = [root] + const seen = new Set() + while (pending.length > 0) { + const value = pending.pop() + if (value === undefined || value === null) continue + if (typeof value === 'string') { + if (value.trim().length > 0) return true + continue + } + if (typeof value === 'object') { + // Profiles are normally serializable trees. Treat a repeated reference as nonempty so a + // cyclic opaque metadata value cannot recurse forever or make an unsupported axis disappear. + if (seen.has(value)) return true + seen.add(value) + for (const child of Array.isArray(value) ? value : Object.values(value)) { + pending.push(child) + } + continue + } + return true + } + return false +} diff --git a/src/mcp/worktree-harness.ts b/src/mcp/worktree-harness.ts index bab5a446..45f4ae7f 100644 --- a/src/mcp/worktree-harness.ts +++ b/src/mcp/worktree-harness.ts @@ -30,6 +30,11 @@ import { type WorkspacePlan, type WorkspacePlanReceipt, } from '@tangle-network/agent-profile-materialize' +import { + assertProfileMaterialization, + profileMaterializationAxes, + worktreeCliProfileMaterialization, +} from '../agent/profile-materialization' import { type CodexExecutionPolicy, type CodexTokenUsage, @@ -152,8 +157,8 @@ export interface RunWorktreeHarnessOptions { repoRoot: string /** * Supervisor-authored prompt/model plus structural resources materialized into the worktree. - * `model.default` selects the one-shot model; `small`, `provider`, and `metadata` remain hints. - * Resource failures are always fatal here, regardless of `resources.failOnError`. + * `model.default` selects the one-shot model. Routing-only model hints and + * `resources.failOnError` are rejected because this path cannot honor them. */ profile: AgentProfile /** Local harness for this run. This explicit choice overrides `profile.harness`. */ @@ -426,12 +431,15 @@ function materializationOnlyProfile(profile: AgentProfile): AgentProfile { } function assertSupportedWorktreeProfile(profile: AgentProfile, harness: LocalHarness): void { - // `profile.harness` is only a preference and the explicit run option wins. Model small/provider/ - // metadata fields are routing or descriptive hints; this fixed one-shot path only selects the - // concrete `model.default`. `resources.failOnError` never weakens this path's fail-closed policy. - const unsupportedAxes = [hasEntries(profile.extensions) ? 'extensions' : null].filter( - (axis): axis is string => axis !== null, - ) + assertProfileMaterialization({ + contract: worktreeCliProfileMaterialization, + changedAxes: profileMaterializationAxes(profile), + context: 'runWorktreeHarness', + }) + // `profile.harness` is only a preference and the explicit run option wins. The contract above + // rejects routing-only model hints and `resources.failOnError`; this harness-specific check + // handles values supported by only a subset of the three local CLIs. + const unsupportedAxes: string[] = [] if (profile.model?.reasoningEffort !== undefined && harness !== 'codex') { unsupportedAxes.push('model.reasoningEffort') } @@ -442,10 +450,6 @@ function assertSupportedWorktreeProfile(profile: AgentProfile, harness: LocalHar } } -function hasEntries(value: object | undefined): boolean { - return value !== undefined && Object.keys(value).length > 0 -} - function assertSafeMaterializedPaths(plan: WorkspacePlan): void { for (const file of plan.files) { if (file.relPath.split('/').some(isGitMetadataSegment)) { diff --git a/src/runtime/router-client.ts b/src/runtime/router-client.ts index 409219e1..7a171d2e 100644 --- a/src/runtime/router-client.ts +++ b/src/runtime/router-client.ts @@ -208,7 +208,7 @@ export async function routerChatWithTools( } // Injected transport short-circuits the network — the offline benchmark seam (see RouterConfig.complete). const raw = cfg.complete - ? await cfg.complete(body) + ? await cfg.complete(structuredClone(body)) : await (async () => { const res = await fetch(`${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions`, { method: 'POST', diff --git a/src/runtime/strategy.ts b/src/runtime/strategy.ts index e9418eb8..f6690207 100644 --- a/src/runtime/strategy.ts +++ b/src/runtime/strategy.ts @@ -477,7 +477,7 @@ function shotExecutor(surface: AgenticSurface, opts: AgenticOptions): Executor( }) const itemSpec = (item: AuthoredHarness): AgentSpec => { - const executor = gateOnDeliverable( - createWorktreeCliExecutor({ - repoRoot: options.repoRoot, - profile: item.profile, - harness: item.harness, - taskPrompt: options.taskPrompt, - ...(item.runId ? { runId: item.runId } : {}), - ...(item.baseRef ? { baseRef: item.baseRef } : {}), - ...(options.testCmd !== undefined ? { testCmd: options.testCmd } : {}), - ...(options.typecheckCmd !== undefined ? { typecheckCmd: options.typecheckCmd } : {}), - ...(options.harnessTimeoutMs !== undefined - ? { harnessTimeoutMs: options.harnessTimeoutMs } - : {}), - ...(options.runGit ? { runGit: options.runGit } : {}), - ...(options.runHarness ? { runHarness: options.runHarness } : {}), - ...(options.runCommand ? { runCommand: options.runCommand } : {}), - }), - deliverable, - ) - return { profile: item.profile, harness: null, executor: executor as AgentSpec['executor'] } + const executorFactory: ExecutorFactory = (_spec, ctx) => { + if (!ctx.node) { + throw new Error('worktreeFanout: supervised node context required') + } + return gateOnDeliverable( + createWorktreeCliExecutor({ + repoRoot: options.repoRoot, + profile: item.profile, + harness: item.harness, + taskPrompt: options.taskPrompt, + executionAttemptId: ctx.node.attemptId, + ...(item.budgetExempt !== undefined ? { budgetExempt: item.budgetExempt } : {}), + ...(item.codexReproducible !== undefined + ? { codexReproducible: item.codexReproducible } + : {}), + ...(item.codexReadDeniedPaths !== undefined + ? { codexReadDeniedPaths: item.codexReadDeniedPaths } + : {}), + ...(item.runId ? { runId: item.runId } : {}), + ...(item.baseRef ? { baseRef: item.baseRef } : {}), + ...(options.testCmd !== undefined ? { testCmd: options.testCmd } : {}), + ...(options.typecheckCmd !== undefined ? { typecheckCmd: options.typecheckCmd } : {}), + ...(options.harnessTimeoutMs !== undefined + ? { harnessTimeoutMs: options.harnessTimeoutMs } + : {}), + ...(options.runGit ? { runGit: options.runGit } : {}), + ...(options.runHarness ? { runHarness: options.runHarness } : {}), + ...(options.runCommand ? { runCommand: options.runCommand } : {}), + }), + deliverable, + ) + } + return { profile: item.profile, harness: null, executorFactory } } const selectWinner = selectValidWinner({ diff --git a/tests/candidate-execution-execute.test.ts b/tests/candidate-execution-execute.test.ts index 9b5078ba..1b433dc6 100644 --- a/tests/candidate-execution-execute.test.ts +++ b/tests/candidate-execution-execute.test.ts @@ -829,7 +829,7 @@ describe('atomic prepared candidate execution', () => { expect(persistedPurposes).not.toContain('benchmark-result') expect(persistedPurposes).not.toContain('trace') expect(persistedPurposes).not.toContain('run-receipt') - }, 15_000) + }) it('allows disposal to retry an unproven pre-claim cleanup', async () => { const fixture = createCandidateExecutionFixture(true) diff --git a/tests/kernel/nested-coordination-durability.test.ts b/tests/kernel/nested-coordination-durability.test.ts new file mode 100644 index 00000000..75297b3f --- /dev/null +++ b/tests/kernel/nested-coordination-durability.test.ts @@ -0,0 +1,139 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { fullProfileMaterialization } from '../../src/agent/profile-materialization' +import type { CoordinationEvent, QuestionRecord } from '../../src/mcp/tools/coordination' +import { supervise } from '../../src/runtime/supervise/supervise' +import type { DriveHarness } from '../../src/runtime/supervise/supervisor-agent' +import { scriptedBrain } from './scripted-brain' + +async function callTool( + url: string, + name: string, + args: Record, +): Promise> { + const response = await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: `${name}-${Math.random()}`, + method: 'tools/call', + params: { name, arguments: args }, + }), + }) + const envelope = (await response.json()) as { + result?: { + structuredContent?: Record + content?: Array<{ type: string; text?: string }> + } + error?: unknown + } + if (!envelope.result) throw new Error(`tool ${name} failed: ${JSON.stringify(envelope.error)}`) + if (envelope.result.structuredContent) return envelope.result.structuredContent + const text = envelope.result.content?.find((entry) => entry.type === 'text')?.text + return text ? (JSON.parse(text) as Record) : {} +} + +function rootBrain() { + const manager = { + name: 'identical-manager', + harness: 'codex', + metadata: { role: 'driver' }, + } + return scriptedBrain([ + { + toolCalls: [ + { + name: 'spawn_agent', + arguments: { profile: manager, task: 'same task', key: 'manager-a' }, + }, + { + name: 'spawn_agent', + arguments: { profile: manager, task: 'same task', key: 'manager-b' }, + }, + ], + }, + { toolCalls: [{ name: 'await_event', arguments: {} }] }, + { toolCalls: [{ name: 'await_event', arguments: {} }] }, + { content: 'finished' }, + ]) +} + +describe('nested supervisor coordination durability', () => { + let runDir: string + + beforeEach(async () => { + runDir = await mkdtemp(join(tmpdir(), 'nested-coordination-')) + }) + + afterEach(async () => { + await rm(runDir, { recursive: true, force: true }) + }) + + it('isolates identical keyed siblings and restores each owner evidence on restart', async () => { + const seenBeforeCurrentQuestion: QuestionRecord[][] = [] + let invocation = 0 + const driveHarness: DriveHarness = async ({ coordinationMcpUrl }) => { + const call = invocation++ + const listed = await callTool(coordinationMcpUrl, 'list_questions', {}) + seenBeforeCurrentQuestion.push((listed.questions ?? []) as QuestionRecord[]) + await callTool(coordinationMcpUrl, 'ask_parent', { + from: 'identical-manager', + level: 'driver', + question: `question from invocation ${call}`, + reason: 'durable owner-isolation proof', + urgency: 'continue-without', + }) + } + const options = { + backend: { + backend: 'router', + routerBaseUrl: 'http://unused.invalid', + routerKey: 'unused', + model: 'unused/model', + } as const, + budget: { maxIterations: 16, maxTokens: 10_000 }, + perWorker: { maxIterations: 4, maxTokens: 1_000 }, + runDir, + runId: 'nested-owner-run', + driveHarness, + driveHarnessMaterialization: fullProfileMaterialization, + maxTurns: 8, + } + const profile = { + name: 'root', + harness: 'cli-base', + prompt: { systemPrompt: 'Run both managers.' }, + } as const + + await supervise(profile, 'root task', { ...options, brain: rootBrain() }) + await supervise(profile, 'root task', { ...options, brain: rootBrain() }) + + expect(seenBeforeCurrentQuestion).toHaveLength(4) + expect(seenBeforeCurrentQuestion.slice(0, 2)).toEqual([[], []]) + for (const prior of seenBeforeCurrentQuestion.slice(2)) { + expect(prior).toHaveLength(1) + expect(prior[0]?.question).toMatch(/^question from invocation [01]$/) + } + + const records = (await readFile(join(runDir, 'coordination-log.jsonl'), 'utf8')) + .trim() + .split('\n') + .map( + (line) => + JSON.parse(line) as { + ownerId?: string + event: CoordinationEvent + }, + ) + .filter((record) => record.event.type === 'question') + const counts = new Map() + for (const record of records) { + if (!record.ownerId) throw new Error('nested coordination record has no owner') + counts.set(record.ownerId, (counts.get(record.ownerId) ?? 0) + 1) + } + expect([...counts.values()].sort()).toEqual([2, 2]) + }) +}) diff --git a/tests/kernel/strategy-evolution.test.ts b/tests/kernel/strategy-evolution.test.ts index 0e419511..010c49f2 100644 --- a/tests/kernel/strategy-evolution.test.ts +++ b/tests/kernel/strategy-evolution.test.ts @@ -20,18 +20,20 @@ import { runStrategyEvolution, selectChampion } from '../../src/runtime/strategy // ── Fixtures ────────────────────────────────────────────────────────────────────── -/** Deterministic surface: score = shots taken on the handle, capped at 2 of 2 — one - * worker pass per shot is observable via tools() calls. Depth (2 shots, one handle) - * scores 1.0; breadth (fresh handle per shot) scores 0.5. */ +/** Deterministic non-degenerate surface: one worker pass per shot is observable via tools() calls. + * Tasks alternate between two- and three-pass targets, so depth beats breadth on every task while + * the paired lift still has real variation for the statistical promotion decision. */ function shotCountingSurface(): AgenticSurface { const shotsByHandle = new Map() + const targetByHandle = new Map() let seq = 0 return { name: 'shot-counter', - async open() { + async open(task) { seq += 1 const id = `h-${seq}` shotsByHandle.set(id, 0) + targetByHandle.set(id, targetForTask(task.id)) return { id, surface: 'shot-counter' } }, async tools(_t, handle) { @@ -42,12 +44,22 @@ function shotCountingSurface(): AgenticSurface { return 'ok' }, async score(_t, handle) { - return { passes: Math.min(shotsByHandle.get(handle.id) ?? 0, 2), total: 2, errored: 0 } + const total = targetByHandle.get(handle.id) ?? 2 + return { + passes: Math.min(shotsByHandle.get(handle.id) ?? 0, total), + total, + errored: 0, + } }, async close() {}, } } +function targetForTask(taskId: string): 2 | 3 { + const suffix = Number(/(\d+)$/u.exec(taskId)?.[1]) + return Number.isFinite(suffix) && suffix % 4 >= 2 ? 3 : 2 +} + const twoShotDepthModule = `import { defineStrategy } from '@tangle-network/agent-runtime/kernel' export default defineStrategy('two-shot-depth', async ({ surface, task, shot }) => { const handle = await surface.open(task) @@ -294,11 +306,12 @@ describe('runStrategyEvolution', () => { import { discriminatingMeans, pickChampion } from '../../src/runtime/strategy-evolution' -/** Difficulty by task id: 'easy-*' tasks score 1.0 for ANY strategy; others score by - * shots-on-handle capped at 2 (depth 1.0, breadth 0.5) — the middle band. */ +/** Difficulty by task id: 'easy-*' tasks score 1.0 for ANY strategy; band tasks alternate + * two- and three-pass targets so depth beats breadth with non-degenerate paired evidence. */ function difficultySurface(): AgenticSurface { const shotsByHandle = new Map() const taskByHandle = new Map() + const targetByHandle = new Map() let seq = 0 return { name: 'difficulty', @@ -307,6 +320,7 @@ function difficultySurface(): AgenticSurface { const id = `h-${seq}` shotsByHandle.set(id, 0) taskByHandle.set(id, task.id) + targetByHandle.set(id, targetForTask(task.id)) return { id, surface: 'difficulty' } }, async tools(_t, handle) { @@ -319,7 +333,12 @@ function difficultySurface(): AgenticSurface { async score(_t, handle) { if (taskByHandle.get(handle.id)?.startsWith('easy-')) return { passes: 2, total: 2, errored: 0 } - return { passes: Math.min(shotsByHandle.get(handle.id) ?? 0, 2), total: 2, errored: 0 } + const total = targetByHandle.get(handle.id) ?? 2 + return { + passes: Math.min(shotsByHandle.get(handle.id) ?? 0, total), + total, + errored: 0, + } }, async close() {}, } diff --git a/tests/kernel/strategy-suite.test.ts b/tests/kernel/strategy-suite.test.ts index 0288a7f3..47dcc287 100644 --- a/tests/kernel/strategy-suite.test.ts +++ b/tests/kernel/strategy-suite.test.ts @@ -658,9 +658,9 @@ describe('promotionGate non-inferiority', () => { const rows = Array.from({ length: 12 }, (_, i) => ({ id: `t${i}`, incScore: 0.6 + (i % 3) * 0.05, - candScore: 0.59 + (i % 3) * 0.05, - incUsd: 0.028, - candUsd: 0.013, + candScore: 0.59 + (i % 3) * 0.05 + (i % 2) * 0.004, + incUsd: 0.027 + (i % 3) * 0.001, + candUsd: 0.012 + (i % 2) * 0.002, })) const v = promotionGate({ report: costReport(rows), @@ -696,7 +696,7 @@ describe('promotionGate non-inferiority', () => { const rows = Array.from({ length: 12 }, (_, i) => ({ id: `t${i}`, incScore: 0.6, - candScore: 0.61, + candScore: 0.608 + (i % 3) * 0.002, incUsd: 0.02, candUsd: i % 2 === 0 ? 0.021 : 0.019, })) @@ -714,7 +714,7 @@ describe('promotionGate non-inferiority', () => { const rows = Array.from({ length: 12 }, (_, i) => ({ id: `t${i}`, incScore: 0.6, - candScore: 0.6, + candScore: 0.598 + (i % 3) * 0.002, incUsd: 0.03, candUsd: 0.012 + (i % 2) * 0.002, })) diff --git a/tests/kernel/supervise-global-concurrency.test.ts b/tests/kernel/supervise-global-concurrency.test.ts new file mode 100644 index 00000000..797ecab3 --- /dev/null +++ b/tests/kernel/supervise-global-concurrency.test.ts @@ -0,0 +1,399 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' +import type { MakeWorkerAgent } from '../../src/mcp/tools/coordination' +import { driverChild } from '../../src/runtime/supervise/driver-executor' +import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { supervise } from '../../src/runtime/supervise/supervise' +import { createSupervisor } from '../../src/runtime/supervise/supervisor' +import { supervisorAgent } from '../../src/runtime/supervise/supervisor-agent' +import type { + Agent, + AgentSpec, + Executor, + ExecutorResult, + Scope, +} from '../../src/runtime/supervise/types' +import { scriptedBrain } from './scripted-brain' + +const zeroCost = { iterations: 1, tokens: { input: 1, output: 1 }, usd: 0, ms: 0 } +const knownZero = { + iterations: 0, + tokens: { input: 0, output: 0 }, + usd: 0, + ms: 0, +} + +interface Activity { + live: number + peak: number +} + +function enter(activity: Activity): void { + activity.live += 1 + activity.peak = Math.max(activity.peak, activity.live) +} + +function leave(activity: Activity): void { + activity.live -= 1 +} + +function trackedLeaf(name: string, activity?: Activity, holdMs = 0): Agent { + const result: ExecutorResult = { + outRef: `leaf:${name}`, + out: { name }, + verdict: { valid: true, score: 1 }, + spent: zeroCost, + } + const executor: Executor = { + runtime: 'router', + async execute(): Promise> { + if (activity) enter(activity) + try { + if (holdMs > 0) await new Promise((resolve) => setTimeout(resolve, holdMs)) + return result + } finally { + if (activity) leave(activity) + } + }, + teardown: () => Promise.resolve({ destroyed: true }), + resultArtifact: () => result, + } + const spec: AgentSpec = { profile: { name }, harness: null, executor } + return { name, act: async () => result.out, executorSpec: spec } as Agent & { + executorSpec: AgentSpec + } +} + +function failingLeaf(name: string): Agent { + const executor: Executor = { + runtime: 'router', + execute: () => Promise.reject(new Error('executor failed')), + teardown: () => Promise.resolve({ destroyed: true }), + // This fake fails before any provider call. Keep the capacity test independent from the + // separate fail-closed rule that unknown token usage consumes the remaining capped budget. + accounting: () => ({ reported: knownZero, reservation: knownZero }), + resultArtifact: () => { + throw new Error('failed executor has no result') + }, + } + const spec: AgentSpec = { profile: { name }, harness: null, executor } + return { name, act: async () => undefined, executorSpec: spec } as Agent & { + executorSpec: AgentSpec + } +} + +function abortableLeaf(name: string): Agent { + const executor: Executor = { + runtime: 'router', + execute(_task, signal): Promise> { + return new Promise((_resolve, reject) => { + const fail = () => { + const error = new Error('aborted') + error.name = 'AbortError' + reject(error) + } + if (signal.aborted) fail() + else signal.addEventListener('abort', fail, { once: true }) + }) + }, + teardown: () => Promise.resolve({ destroyed: true }), + accounting: () => ({ reported: knownZero, reservation: knownZero }), + resultArtifact: () => { + throw new Error('aborted executor has no result') + }, + } + const spec: AgentSpec = { profile: { name }, harness: null, executor } + return { name, act: async () => undefined, executorSpec: spec } as Agent & { + executorSpec: AgentSpec + } +} + +function profileDepth(profile: AgentProfile): number { + const value = profile.metadata?.depth + if (typeof value !== 'number') + throw new Error(`profile ${profile.name ?? ''} has no depth`) + return value +} + +describe('supervise tree-wide worker capacity', () => { + it('retains a slot when teardown cannot prove the executor was destroyed', async () => { + let secondReason: string | undefined + const unkillable = trackedLeaf('placeholder') as Agent & { + executorSpec: AgentSpec + } + unkillable.executorSpec = { + profile: { name: 'unkillable' }, + harness: null, + executor: { + runtime: 'router', + execute: () => new Promise>(() => {}), + teardown: () => new Promise(() => {}), + resultArtifact: () => { + throw new Error('unkillable executor has no result') + }, + }, + } + const root: Agent = { + name: 'root', + async act(task, scope): Promise { + const first = scope.spawn(unkillable, task, { + budget: { maxIterations: 1, maxTokens: 10, deadlineMs: 5 }, + label: 'unkillable', + }) + expect(first.ok).toBe(true) + expect((await scope.next())?.kind).toBe('down') + const second = scope.spawn(() => trackedLeaf('replacement'), task, { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'replacement', + }) + secondReason = second.ok ? 'accepted' : second.reason + return 'finished' + }, + } + + const result = await createSupervisor().run(root, 'task', { + budget: { maxIterations: 2, maxTokens: 20 }, + maxLiveWorkers: 1, + runId: 'retain-unconfirmed-capacity', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }) + + expect(secondReason).toBe('max-live-workers') + expect(result.kind).toBe('no-winner') + }) + + it('holds one cap across root → manager → sub-manager → worker execution', async () => { + const cap = 5 + const activity: Activity = { live: 0, peak: 0 } + const constructedDepths: number[] = [] + const journal = new InMemorySpawnJournal() + const blobs = new InMemoryResultBlobStore() + + let makeWorkerAgent: MakeWorkerAgent + makeWorkerAgent = (profile, context) => { + const depth = profileDepth(profile) + constructedDepths.push(depth) + if (profile.metadata?.role !== 'driver') + return trackedLeaf(profile.name ?? 'leaf', activity, 50) + + const childProfiles: AgentProfile[] = + depth === 1 + ? [ + { + name: `${profile.name}-sub-manager`, + harness: 'cli-base', + metadata: { role: 'driver', depth: 2 }, + }, + ] + : [0, 1].map((index) => ({ + name: `${profile.name}-leaf-${index}`, + harness: 'cli-base', + metadata: { role: 'worker', depth: 3 }, + })) + const brain = scriptedBrain([ + { + toolCalls: childProfiles.map((child) => ({ + name: 'spawn_agent', + arguments: { profile: child, task: `run ${child.name}` }, + })), + }, + { toolCalls: [{ name: 'await_event', arguments: {} }] }, + { content: 'managed' }, + ]) + const nested = supervisorAgent(profile, { + blobs, + makeWorkerAgent, + perWorker: + depth === 1 + ? { maxIterations: 40, maxTokens: 40_000 } + : { maxIterations: 10, maxTokens: 10_000 }, + brain, + maxTurns: 6, + }) + const tracked: Agent = { + name: nested.name, + async act(task, scope): Promise { + enter(activity) + try { + return await nested.act(task, scope) + } finally { + leave(activity) + } + }, + } + return driverChild(profile, tracked, journal, context?.execution) + } + + const rootBrain = scriptedBrain([ + { + toolCalls: [0, 1].map((index) => ({ + name: 'spawn_agent', + arguments: { + profile: { + name: `manager-${index}`, + harness: 'cli-base', + metadata: { role: 'driver', depth: 1 }, + }, + task: `run branch ${index}`, + }, + })), + }, + { toolCalls: [{ name: 'await_event', arguments: {} }] }, + { toolCalls: [{ name: 'await_event', arguments: {} }] }, + { content: 'done' }, + ]) + + const result = await supervise( + { name: 'root', harness: 'cli-base' }, + 'run a three-level tree', + { + budget: { maxIterations: 500, maxTokens: 500_000 }, + perWorker: { maxIterations: 160, maxTokens: 160_000 }, + maxDepth: 5, + maxLiveWorkers: cap, + makeWorkerAgent, + brain: rootBrain, + journal, + blobs, + runId: 'global-cap', + }, + ) + + expect(result.kind).toBe('winner') + expect(activity.live).toBe(0) + expect(activity.peak).toBe(cap) + expect(constructedDepths).toHaveLength(cap) + expect(constructedDepths.filter((depth) => depth === 1)).toHaveLength(2) + expect(constructedDepths.filter((depth) => depth === 2)).toHaveLength(2) + expect(constructedDepths).toContain(3) + + const trees = ( + journal as unknown as { trees: Map }> } + ).trees + const spawnedIds = [...trees.values()] + .flatMap((tree) => tree.events) + .filter((event) => event.kind === 'spawned') + .map((event) => event.id) + expect(spawnedIds.some((id) => id.split(':s').length === 4)).toBe(true) + }) + + it('releases once on construction failure, budget refusal, completion, and completed-key replay', async () => { + const seen: Record = {} + let completedFactoryCalls = 0 + const root: Agent = { + name: 'root', + async act(task, scope: Scope): Promise { + try { + scope.spawn( + () => { + throw new Error('construction failed') + }, + task, + { budget: { maxIterations: 1, maxTokens: 10 }, label: 'bad-factory' }, + ) + } catch (error) { + seen.constructionError = error instanceof Error ? error.message : String(error) + } + seen.afterConstructionError = scope.workerCapacity.live + + try { + scope.spawn(() => trackedLeaf('invalid-budget'), task, { + budget: { maxIterations: -1, maxTokens: 10 }, + label: 'invalid-budget', + }) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + seen.invalidBudgetRefused = message.includes('non-negative safe integer') + } + seen.afterInvalidBudget = scope.workerCapacity.live + + const tooLarge = scope.spawn(() => trackedLeaf('too-large'), task, { + budget: { maxIterations: 101, maxTokens: 100_001 }, + label: 'too-large', + }) + seen.tooLarge = tooLarge.ok ? 'accepted' : tooLarge.reason + seen.afterBudgetRefusal = scope.workerCapacity.live + + const failed = scope.spawn(() => failingLeaf('failed'), task, { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'failed', + }) + seen.failedStarted = failed.ok + seen.failedSettled = (await scope.next())?.kind + seen.afterExecutorFailure = scope.workerCapacity.live + + const aborted = scope.spawn(() => abortableLeaf('aborted'), task, { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'aborted', + }) + seen.abortedStarted = aborted.ok + if (aborted.ok) aborted.handle.abort('test abort') + seen.abortedSettled = (await scope.next())?.kind + seen.afterAbort = scope.workerCapacity.live + + const first = scope.spawn(() => trackedLeaf('keyed'), task, { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'keyed', + key: 'assignment', + }) + seen.first = first.ok + seen.duringRun = scope.workerCapacity.live + seen.settled = (await scope.next())?.kind + seen.afterCompletion = scope.workerCapacity.live + + const replay = scope.spawn( + () => { + completedFactoryCalls += 1 + return trackedLeaf('keyed') + }, + task, + { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'keyed', + key: 'assignment', + }, + ) + seen.replay = replay.ok ? replay.prior?.state : replay.reason + seen.afterReplay = scope.workerCapacity.live + return 'done' + }, + } + + const result = await createSupervisor().run(root, 'task', { + budget: { maxIterations: 100, maxTokens: 100_000 }, + maxLiveWorkers: 1, + runId: 'capacity-release', + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }) + + expect(result.kind).toBe('winner') + expect(seen).toEqual({ + constructionError: 'construction failed', + afterConstructionError: 0, + invalidBudgetRefused: true, + afterInvalidBudget: 0, + tooLarge: 'budget-exhausted', + afterBudgetRefusal: 0, + failedStarted: true, + failedSettled: 'down', + afterExecutorFailure: 0, + abortedStarted: true, + abortedSettled: 'down', + afterAbort: 0, + first: true, + duringRun: 1, + settled: 'done', + afterCompletion: 0, + replay: 'completed', + afterReplay: 0, + }) + // Reuse prepares the requested profile/task so the semantic key cannot alias different work. + // It still constructs no executor, reserves no budget, and runs nothing. + expect(completedFactoryCalls).toBe(1) + }) +}) diff --git a/tests/kernel/worktree-loop.test.ts b/tests/kernel/worktree-loop.test.ts index 14e5c052..e82735c5 100644 --- a/tests/kernel/worktree-loop.test.ts +++ b/tests/kernel/worktree-loop.test.ts @@ -41,6 +41,12 @@ const okHarness = vi.fn(async () => ({ killedBySignal: null as NodeJS.Signals | null, durationMs: 1, timedOut: false, + usage: { + inputTokens: 10, + cachedInputTokens: 0, + outputTokens: 5, + reasoningOutputTokens: 0, + }, })) describe('worktreeLoopRunner — the migrated generic coder path', () => { @@ -51,8 +57,18 @@ describe('worktreeLoopRunner — the migrated generic coder path', () => { taskPrompt: 'fix the off-by-one', budget, harnesses: [ - { name: 'claude', profile: profile('claude'), harness: 'claude' }, - { name: 'opencode', profile: profile('opencode'), harness: 'opencode' }, + { + name: 'claude', + profile: profile('claude'), + harness: 'claude', + budgetExempt: false, + }, + { + name: 'opencode', + profile: profile('opencode'), + harness: 'opencode', + budgetExempt: false, + }, ], testCmd: 'pnpm test', typecheckCmd: 'pnpm typecheck', @@ -76,7 +92,14 @@ describe('worktreeLoopRunner — the migrated generic coder path', () => { repoRoot: '/repo', taskPrompt: 'fix it', budget, - harnesses: [{ name: 'claude', profile: profile('claude'), harness: 'claude' }], + harnesses: [ + { + name: 'claude', + profile: profile('claude'), + harness: 'claude', + budgetExempt: false, + }, + ], testCmd: 'pnpm test', require: ['tests'], runGit: fakeGitWith(() => ({ @@ -95,7 +118,14 @@ describe('worktreeLoopRunner — the migrated generic coder path', () => { repoRoot: '/repo', taskPrompt: 'do nothing', budget, - harnesses: [{ name: 'claude', profile: profile('claude'), harness: 'claude' }], + harnesses: [ + { + name: 'claude', + profile: profile('claude'), + harness: 'claude', + budgetExempt: false, + }, + ], runGit: fakeGitWith(() => ({ patch: '', shortstat: ' 0 files changed\n' })), runHarness: okHarness, }) diff --git a/tests/mcp/worktree-harness.test.ts b/tests/mcp/worktree-harness.test.ts index 48bba6bb..5420ade6 100644 --- a/tests/mcp/worktree-harness.test.ts +++ b/tests/mcp/worktree-harness.test.ts @@ -200,7 +200,6 @@ describe('runWorktreeHarness profile materialization', () => { name: 'resource-instructions', content: resourceInstructionMarker, }, - failOnError: true, }, } @@ -369,12 +368,7 @@ describe('runWorktreeHarness profile materialization', () => { repoRoot, profile: { harness: 'codex', - model: { - default: 'claude-model', - small: 'small-routing-hint', - provider: 'anthropic', - metadata: { tier: 'research' }, - }, + model: { default: 'claude-model' }, prompt: { systemPrompt: 'DIRECT_SYSTEM_a3563f03' }, resources: { files: [ @@ -417,9 +411,6 @@ describe('runWorktreeHarness profile materialization', () => { expect(options.harness).toBe('claude') expect(options.invocation?.command).toBe('claude') expect(options.invocation?.args).toContain('claude-model') - expect(options.invocation?.args).not.toContain('small-routing-hint') - expect(options.invocation?.args).not.toContain('anthropic') - expect(options.invocation?.args).not.toContain('research') expect(readFileSync(join(options.cwd, paths.file), 'utf8')).toContain( 'FILE_MARKER_5570e069', ) @@ -485,7 +476,6 @@ describe('runWorktreeHarness profile materialization', () => { repoRoot, profile: { resources: { - failOnError: false, files: [ { path: '.agent-profile/remote.txt', @@ -539,7 +529,7 @@ describe('runWorktreeHarness profile materialization', () => { } }) - it('rejects unsupported behavior axes before worker launch and removes the worktree', async () => { + it('rejects unsupported behavior axes before worker launch or worktree creation', async () => { const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) const runId = 'unsupported-behavior-axes' const runHarness = vi.fn() @@ -548,11 +538,14 @@ describe('runWorktreeHarness profile materialization', () => { runWorktreeHarness({ repoRoot, profile: { - tools: { shell: false }, - permissions: { shell: 'deny' }, + model: { + small: 'routing-model', + provider: 'provider-hint', + metadata: { tier: 'research' }, + }, connections: [{ connectionId: 'connection-1', capabilities: ['read'] }], + resources: { failOnError: false }, confidential: { tee: 'any' }, - modes: { review: { prompt: 'Review only.' } }, extensions: { codex: { feature: true } }, }, harness: 'codex', @@ -561,7 +554,7 @@ describe('runWorktreeHarness profile materialization', () => { runHarness, }), ).rejects.toThrow( - /unsupported worktree behavior: tools, permissions, connections, confidential, modes, extensions/u, + /profile materialization would drop axis changes.*modelSmall, modelProvider, modelMetadata, connections, resourceFailOnError, confidential, extensions/su, ) expect(runHarness).not.toHaveBeenCalled() expect(existsSync(join(repoRoot, '.agent-worktrees', runId))).toBe(false) @@ -575,7 +568,9 @@ describe('runWorktreeHarness profile materialization', () => { const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) const runId = 'run-and-cleanup-failures' const worktreePath = join(repoRoot, '.agent-worktrees', runId) - const runHarness = vi.fn() + const runHarness = vi.fn(async () => { + throw new Error('simulated harness failure') + }) let interceptedRemoval = false const runGit: GitRunner = (args, { cwd }) => { if (args[0] === 'worktree' && args[1] === 'remove' && !interceptedRemoval) { @@ -595,7 +590,7 @@ describe('runWorktreeHarness profile materialization', () => { try { await runWorktreeHarness({ repoRoot, - profile: { tools: { shell: false } }, + profile: {}, harness: 'codex', taskPrompt: 'task', runId, @@ -608,12 +603,12 @@ describe('runWorktreeHarness profile materialization', () => { expect(error).toBeInstanceOf(AggregateError) const errors = (error as AggregateError).errors as Error[] - expect(errors[0]?.message).toContain('unsupported worktree behavior: tools') + expect(errors[0]?.message).toContain('simulated harness failure') expect(errors[1]).toBeInstanceOf(AggregateError) const cleanupErrors = (errors[1] as AggregateError).errors as Error[] expect(cleanupErrors[0]?.message).toContain('worktree remove') expect(cleanupErrors[1]?.message).toContain('branch -D') - expect(runHarness).not.toHaveBeenCalled() + expect(runHarness).toHaveBeenCalledOnce() expect(interceptedRemoval).toBe(true) expect(existsSync(worktreePath)).toBe(true) } finally { @@ -623,7 +618,7 @@ describe('runWorktreeHarness profile materialization', () => { } }) - it('rejects nested controls the pinned materializer would silently drop', async () => { + it('rejects harness-specific controls the current materializer cannot preserve', async () => { const repoRoot = initializeRepository({ 'src/value.ts': 'export const value = 1\n' }) const runHarness = vi.fn() const cases: Array<{ @@ -636,13 +631,6 @@ describe('runWorktreeHarness profile materialization', () => { runId: 'codex-nested-controls', harness: 'codex', profile: { - mcp: { - disabled: { - command: 'node', - enabled: false, - headers: { Authorization: 'redacted' }, - }, - }, subagents: { helper: { prompt: 'Help.', @@ -653,11 +641,9 @@ describe('runWorktreeHarness profile materialization', () => { }, }, dropped: [ - 'mcp["disabled"].enabled', - 'mcp["disabled"].headers', - 'subagents["helper"].permissions', - 'subagents["helper"].maxSteps', - 'subagents["helper"].tools', + 'subagent "helper" does not support tools', + 'subagent "helper" does not support permissions', + 'subagent "helper" does not support maxSteps', ], }, { @@ -665,21 +651,14 @@ describe('runWorktreeHarness profile materialization', () => { harness: 'claude', profile: { model: { reasoningEffort: 'high' }, - hooks: { - PreToolUse: [{ command: 'node hook.mjs', env: { MODE: 'strict' }, blocking: false }], - }, }, - dropped: [ - 'model.reasoningEffort', - 'hooks["PreToolUse"][0].env', - 'hooks["PreToolUse"][0].blocking', - ], + dropped: ['model.reasoningEffort'], }, { runId: 'opencode-nested-controls', harness: 'opencode', profile: { mcp: { local: { command: 'node', cwd: 'required-directory' } } }, - dropped: ['mcp["local"].cwd'], + dropped: ['MCP server "local" does not support a cwd field'], }, ] diff --git a/tests/profile-materialization.test.ts b/tests/profile-materialization.test.ts index fa37907d..beed187b 100644 --- a/tests/profile-materialization.test.ts +++ b/tests/profile-materialization.test.ts @@ -1,14 +1,185 @@ +import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { assertProfileMaterialization, + controlProfileMaterialization, defineProfileMaterializationContract, + fullProfileMaterialization, + profileMaterializationAxes, + promptModelProfileMaterialization, promptOnlyProfileMaterialization, promptResourceProfileMaterialization, sandboxActProfileMaterialization, validateProfileMaterialization, + worktreeCliProfileMaterialization, } from '../src/agent' describe('profile materialization contracts', () => { + it('maps a complete profile to every exact nonempty canonical axis it requests', () => { + const profile: AgentProfile = { + name: 'researcher', + description: 'Tests competing mechanisms', + version: '1', + tags: ['science'], + prompt: { systemPrompt: 'Run discriminating experiments.' }, + model: { default: 'provider/model', reasoningEffort: 'high' }, + harness: 'codex', + permissions: { shell: 'ask' }, + tools: { web: true }, + mcp: { papers: { transport: 'http', url: 'https://papers.example.test/mcp' } }, + connections: [{ connectionId: 'literature', capabilities: ['search'] }], + subagents: { critic: { prompt: 'Find confounds.' } }, + resources: { + skills: [{ kind: 'inline', name: 'hypothesis', content: 'Test mechanisms.' }], + }, + hooks: { afterTool: [{ command: './capture-result' }] }, + modes: { adversarial: { prompt: 'Try to falsify the claim.' } }, + confidential: { sealed: true }, + metadata: { role: 'driver' }, + extensions: { codex: { sandbox: 'workspace-write' } }, + } + + expect(profileMaterializationAxes(profile)).toEqual([ + 'name', + 'description', + 'version', + 'tags', + 'systemPrompt', + 'modelDefault', + 'modelReasoningEffort', + 'harness', + 'permissions', + 'tools', + 'mcp', + 'connections', + 'subagents', + 'skills', + 'hooks', + 'modes', + 'confidential', + 'metadata', + 'extensions', + ]) + }) + + it('omits empty structure while retaining explicit false and zero requests', () => { + expect( + profileMaterializationAxes({ + name: ' ', + tags: [], + prompt: { systemPrompt: '', instructions: [' '] }, + model: { metadata: {} }, + permissions: {}, + tools: { web: false }, + mcp: {}, + connections: [], + subagents: {}, + resources: { files: [], failOnError: false }, + hooks: { afterTool: [] }, + modes: {}, + confidential: { sealed: false }, + metadata: { retries: 0 }, + extensions: { codex: undefined }, + }), + ).toEqual(['tools', 'resourceFailOnError', 'confidential', 'metadata']) + }) + + it('treats cyclic opaque metadata as a nonempty request without recursing forever', () => { + const metadata: Record = {} + metadata.self = metadata + + expect(profileMaterializationAxes({ metadata })).toEqual(['metadata']) + }) + + it('handles deeply nested empty opaque metadata without exhausting the call stack', () => { + const metadata: Record = {} + let cursor = metadata + for (let index = 0; index < 25_000; index += 1) { + const next: Record = {} + cursor.next = next + cursor = next + } + + expect(profileMaterializationAxes({ metadata })).toEqual([]) + }) + + it('separates prompt-and-model execution from full-profile execution', () => { + const requested = profileMaterializationAxes({ + name: 'router-worker', + prompt: { systemPrompt: 'Solve it.' }, + model: { default: 'provider/model' }, + tools: { shell: true }, + harness: 'codex', + metadata: { authorizationId: 'auth-1' }, + }) + + expect( + validateProfileMaterialization({ + contract: promptModelProfileMaterialization, + changedAxes: requested, + }).map((issue) => issue.axis), + ).toEqual(['tools']) + expect( + validateProfileMaterialization({ + contract: fullProfileMaterialization, + changedAxes: requested, + }), + ).toEqual([]) + }) + + it('does not let a limited path claim model or prompt fields it cannot apply', () => { + const requested = profileMaterializationAxes({ + name: 'pi-worker', + prompt: { systemPrompt: 'Solve it.', instructions: ['Show evidence.'] }, + model: { + default: 'provider/model', + small: 'provider/small', + reasoningEffort: 'high', + }, + harness: 'pi', + metadata: { run: 'one' }, + }) + + expect( + validateProfileMaterialization({ + contract: promptModelProfileMaterialization, + changedAxes: requested, + }).map((issue) => issue.axis), + ).toEqual(['modelSmall', 'modelReasoningEffort']) + expect( + validateProfileMaterialization({ + contract: controlProfileMaterialization, + changedAxes: requested, + }).map((issue) => issue.axis), + ).toEqual([ + 'systemPrompt', + 'instructions', + 'modelDefault', + 'modelSmall', + 'modelReasoningEffort', + ]) + }) + + it('describes the local worktree CLI without claiming placement or ignored profile fields', () => { + expect( + validateProfileMaterialization({ + contract: worktreeCliProfileMaterialization, + changedAxes: [ + 'name', + 'systemPrompt', + 'modelDefault', + 'modelSmall', + 'tools', + 'permissions', + 'connections', + 'confidential', + 'extensions', + 'resourceFailOnError', + ], + }).map((issue) => issue.axis), + ).toEqual(['modelSmall', 'connections', 'confidential', 'extensions', 'resourceFailOnError']) + }) + it('lets a broad prompt contract carry prompt sub-axes', () => { expect( validateProfileMaterialization({ @@ -60,6 +231,7 @@ describe('profile materialization contracts', () => { changedAxes: [ 'name', 'model', + 'harness', 'systemPrompt', 'files', 'skills', diff --git a/tests/runtime/worktree-cli-executor.test.ts b/tests/runtime/worktree-cli-executor.test.ts index 2c425d49..094193ed 100644 --- a/tests/runtime/worktree-cli-executor.test.ts +++ b/tests/runtime/worktree-cli-executor.test.ts @@ -295,7 +295,7 @@ describe('createWorktreeCliExecutor', () => { runGit: makeFakeGit(state), runHarness: vi.fn(), }), - ).toThrow(/profile cannot be materialized.*connections/) + ).toThrow(/profile materialization would drop axis changes.*connections/s) expect(state.worktreesCreated).toEqual([]) expect(state.worktreesRemoved).toEqual([]) }) From a603d8fa86dd541184646e4f5f6c37ce7f6519ac Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 06:17:33 -0600 Subject: [PATCH 07/12] fix(supervise): preserve root deadline cause --- src/runtime/supervise/supervisor.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/runtime/supervise/supervisor.ts b/src/runtime/supervise/supervisor.ts index b54657af..1ec142f0 100644 --- a/src/runtime/supervise/supervisor.ts +++ b/src/runtime/supervise/supervisor.ts @@ -593,6 +593,13 @@ export function createSupervisor(): Supervisor { actOutcome = { ok: false, error } } finally { executionAborted = controller.signal.aborted + // A child inherits the root cutoff and can settle the root act on that exact timer turn. + // If its settlement callback runs before the root timer callback, the finally block clears + // the still-pending root timer. Preserve the deadline cause from the shared absolute clock; + // do not overwrite a caller/breaker abort that already won the controller race. + if (!controller.signal.aborted && rootDeadlineAtMs > 0 && now() >= rootDeadlineAtMs) { + deadlineExceeded = true + } clearRootDeadline?.() // Join barrier: tear down every still-live child. Generalizes the kernel's // `finally{ Promise.allSettled(destroy) }` — a teardown throw is allSettled'd and From 70ed7e93eac19b686db99d57ea38cd57e8fb988a Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 06:36:30 -0600 Subject: [PATCH 08/12] fix(supervise): derive failure guidance from traces --- src/mcp/tools/checks.ts | 33 +++++- src/runtime/supervise-surface.ts | 140 +++++++++++++++++++----- src/runtime/supervise/trace-evidence.ts | 15 +++ tests/kernel/checks.test.ts | 63 +++++++++++ tests/kernel/supervise-surface.test.ts | 122 +++++++++++++++++++++ 5 files changed, 341 insertions(+), 32 deletions(-) create mode 100644 tests/kernel/supervise-surface.test.ts diff --git a/src/mcp/tools/checks.ts b/src/mcp/tools/checks.ts index f22654dd..a1c02db3 100644 --- a/src/mcp/tools/checks.ts +++ b/src/mcp/tools/checks.ts @@ -21,8 +21,10 @@ import { type AnalystSeverity, type EvidenceRef, makeFinding, + type TraceAnalysisStore, } from '@tangle-network/agent-eval' import { assertTraceDerivedFindings } from '../../runtime' +import { isTraceAnalysisStore } from '../../runtime/supervise/trace-evidence' // agent-eval's root entry exports the lift (`makeFinding`) + the `AnalystFinding`/`EvidenceRef` shapes // but NOT the raw-row validator / schema prompt (kind-factory-internal). We inline a minimal, @@ -179,9 +181,13 @@ function evidenceRefs(uri: string, excerpt?: string): EvidenceRef[] { return [{ kind, uri, ...(excerpt ? { excerpt } : {}) }] } -/** Render a worker's trace (tool calls + results) into the text an analyst lens reads. Generic over - * the trace shape: a `{ messages }` conversation, a bare message array, else stringified. */ -export function renderTrace(trace: unknown): string { +/** Render a worker's trace (tool calls + results) into the text an analyst lens reads. Conversation + * values remain synchronous; the official bounded store is read asynchronously through its own + * overview and view operations instead of stringifying its prototype to `{}`. */ +export function renderTrace(trace: TraceAnalysisStore): Promise +export function renderTrace(trace: unknown): string +export function renderTrace(trace: unknown): string | Promise { + if (isTraceAnalysisStore(trace)) return renderTraceStore(trace) const messages = Array.isArray(trace) ? trace : trace && @@ -207,6 +213,24 @@ export function renderTrace(trace: unknown): string { .slice(0, 8000) } +async function renderTraceStore(trace: TraceAnalysisStore): Promise { + const overview = await trace.getOverview() + const views = await Promise.all( + overview.sample_trace_ids.slice(0, 3).map(async (traceId) => + trace.viewTrace({ + trace_id: traceId, + per_attribute_byte_cap: 2048, + }), + ), + ) + return [ + `OVERVIEW ${JSON.stringify(overview)}`, + ...views.map((view) => `TRACE ${view.trace_id} ${JSON.stringify(view)}`), + ] + .join('\n') + .slice(0, 8000) +} + export interface CheckRunnerOptions { routerBaseUrl: string routerKey: string @@ -226,7 +250,8 @@ export async function runCheck( const sys = `You are a trace analyst applying ONE lens: look for ${kind.lookFor}\n\n` + `${FINDING_SCHEMA_PROMPT}\n\nReturn ONLY a fenced \`\`\`json array of finding objects (possibly empty).` - const user = `WORKER TRACE:\n${renderTrace(trace)}\n\nApply your lens and emit the findings array.` + const renderedTrace = await Promise.resolve(renderTrace(trace)) + const user = `WORKER TRACE:\n${renderedTrace}\n\nApply your lens and emit the findings array.` const chat = opts.chat ?? defaultChat(opts) const content = await chat(sys, user) const match = content.match(/```(?:json)?\s*([\s\S]*?)```/) diff --git a/src/runtime/supervise-surface.ts b/src/runtime/supervise-surface.ts index 21299ed0..0ffc9523 100644 --- a/src/runtime/supervise-surface.ts +++ b/src/runtime/supervise-surface.ts @@ -13,6 +13,7 @@ * the right home for "supervise over a graded surface". The within-run self-improvement is the analyst * (authored content, swap `analysts`); the across-run kind wraps this call in `improve()`. */ +import { OUTPUT_VALUE, type TraceAnalysisStore } from '@tangle-network/agent-eval' import type { AgentProfile } from '@tangle-network/agent-interface' import type { AnalystRegistry, MakeWorkerAgent } from '../mcp/tools/coordination' import type { RouterConfig } from './router-client' @@ -26,6 +27,8 @@ import { import type { DeliverableSpec } from './supervise/completion-gate' import { supervise } from './supervise/supervise' import type { SupervisorProfile } from './supervise/supervisor-agent' +import { isTraceAnalysisStore } from './supervise/trace-evidence' +import { createPushTraceSource, type TraceSource } from './supervise/trace-source' import type { Agent, AgentSpec, Budget, Executor, ExecutorResult, Spend } from './supervise/types' /** What a surface worker settles with — the surface verdict the driver + deliverable read. `resolved` is @@ -39,35 +42,50 @@ export interface SurfaceWorkerOut { readonly failing?: readonly string[] } -/** Remember the worker's LAST `run_tests` output so the analyst can name the still-failing tests — a - * transparent passthrough for every other surface call. Local to this module (no surface-zoo concept). */ -function captureFailures(base: AgenticSurface): { +/** Instrument every real surface call with the shared push trace source. The last test report remains + * available on `SurfaceWorkerOut` for compatibility, but analysts read only the persisted spans. */ +export function traceSurfaceCalls(base: AgenticSurface): { surface: AgenticSurface failing: () => string[] + traceSource: TraceSource } { let lastReport = '' + const trace = createPushTraceSource() const surface: AgenticSurface = { name: base.name, open: (t) => base.open(t), tools: (t, h) => base.tools(t, h), async call(h, name, args) { - const out = await base.call(h, name, args) - if (name === 'run_tests') lastReport = out - return out + const startedAt = Date.now() + const recordedArgs = structuredClone(args) + try { + const out = await base.call(h, name, args) + trace.record({ + toolName: name, + args: recordedArgs, + result: out, + status: out.startsWith('ERROR:') ? 'error' : 'ok', + startedAt, + endedAt: Date.now(), + }) + if (name === 'run_tests') lastReport = out + return out + } catch (error) { + trace.record({ + toolName: name, + args: recordedArgs, + result: `ERROR: ${error instanceof Error ? error.message : String(error)}`, + status: 'error', + startedAt, + endedAt: Date.now(), + }) + throw error + } }, score: (t, h) => base.score(t, h), close: (h) => base.close(h), } - const failing = () => { - const body = /FAILING:\s*(.+)/i.exec(lastReport)?.[1] - return body - ? body - .split(',') - .map((s) => s.trim()) - .filter(Boolean) - : [] - } - return { surface, failing } + return { surface, failing: () => failingTestNames(lastReport), traceSource: trace.source } } /** The default self-improvement LENS — authored content, not a code path. On each settled worker it hands @@ -82,22 +100,87 @@ export function failuresAnalyst(): AnalystRegistry { area: 'progress', }, ], - run: async (_kindId: string, trace: unknown) => { - const w = (trace ?? {}) as Partial - if (!(typeof w === 'object' && w !== null && 'resolved' in w)) - return { summary: `worker produced: ${JSON.stringify(trace).slice(0, 300)}` } - if (w.resolved) return { summary: 'worker RESOLVED — every check passed; stop.' } - const failing = (w.failing ?? []) as readonly string[] - const head = `worker did NOT resolve — score ${(100 * (w.score ?? 0)).toFixed(0)}%, ${w.shots ?? '?'} shot(s)` + run: async (_kindId: string, trace: TraceAnalysisStore) => { + if (!isTraceAnalysisStore(trace)) return missingRunTestsEvidence() + const report = await latestRunTestsReport(trace) + if (report === undefined) return missingRunTestsEvidence() + const failing = failingTestNames(report) return { summary: failing.length - ? `${head}. STILL FAILING (${failing.length}): ${failing.slice(0, 12).join(', ')}. Spawn the next worker to fix exactly these; if a test keeps failing across workers, give it concrete guidance about that case.` - : `${head}. (no failing-test list available this round)`, + ? `Latest structured run_tests evidence reports STILL FAILING (${failing.length}): ${failing.join(', ')}. Spawn the next worker to fix exactly these; if a test keeps failing across workers, give it concrete guidance about that case.` + : allTestsPassed(report) + ? 'Latest structured run_tests evidence reports every test passed; stop.' + : `Latest structured run_tests evidence contains no parseable failing-test names. Refusing to infer them from worker prose. run_tests output: ${report.slice(0, 300)}`, } }, } } +async function latestRunTestsReport(store: TraceAnalysisStore): Promise { + const overview = await store.getOverview({ tool_names: ['run_tests'] }) + const candidates: Array<{ output: string; endedAt: string; ordinal: number }> = [] + let ordinal = 0 + for (const traceId of overview.sample_trace_ids) { + const view = await store.viewTrace({ trace_id: traceId, per_attribute_byte_cap: 16_384 }) + let spans = view.spans + if (spans === undefined) { + const matches = await store.searchTrace({ + trace_id: traceId, + regex_pattern: 'run_tests', + max_matches: 100, + }) + const spanIds = [ + ...new Set( + matches.hits.filter((hit) => hit.span_name === 'run_tests').map((hit) => hit.span_id), + ), + ] + spans = spanIds.length + ? ( + await store.viewSpans({ + trace_id: traceId, + span_ids: spanIds, + per_attribute_byte_cap: 16_384, + }) + ).spans + : [] + } + for (const span of spans) { + if (span.tool_name !== 'run_tests') continue + const output = span.attributes[OUTPUT_VALUE] + if (typeof output !== 'string') continue + candidates.push({ output, endedAt: span.end_time, ordinal: ordinal++ }) + } + } + candidates.sort( + (left, right) => + Date.parse(left.endedAt) - Date.parse(right.endedAt) || left.ordinal - right.ordinal, + ) + return candidates.at(-1)?.output +} + +function failingTestNames(report: string): string[] { + const body = /FAILING:\s*([^\n]+)/iu.exec(report)?.[1] + if (body === undefined) return [] + return body + .replace(/\.\s+COLLECTION-BLOCKED:.*$/iu, '') + .replace(/\s*\(\+\d+\s+more\)\s*$/iu, '') + .split(',') + .map((name) => name.trim()) + .filter(Boolean) +} + +function allTestsPassed(report: string): boolean { + const fraction = /(\d+)\s*\/\s*(\d+)\s+tests?\s+passed/iu.exec(report) + return fraction !== null && Number(fraction[1]) === Number(fraction[2]) +} + +function missingRunTestsEvidence(): { summary: string } { + return { + summary: + 'Missing structured run_tests span evidence. Refusing to infer failing-test names from worker prose.', + } +} + /** How a worker runs the surface task (its router substrate + per-attempt bounds). */ export interface SurfaceWorkerConfig { readonly routerBaseUrl: string @@ -119,6 +202,7 @@ function surfaceWorkerExecutor( strategy: Strategy, ): Executor { let artifact: ExecutorResult | undefined + const traced = traceSurfaceCalls(surface) return { runtime: 'surface-worker', async execute(brief: unknown): Promise> { @@ -129,9 +213,8 @@ function surfaceWorkerExecutor( systemPrompt: `${task.systemPrompt ?? ''}\n\n— Supervisor guidance for THIS attempt (incorporate it; do not just repeat a prior approach) —\n${guidance}`, } : task - const cap = captureFailures(surface) const r = await runAgentic({ - surface: cap.surface, + surface: traced.surface, task: attemptTask, strategy, budget: worker.budget ?? 1, @@ -146,7 +229,7 @@ function surfaceWorkerExecutor( score: r.score, shots: r.shots, summary: `${strategy.name} ${r.shots} shot(s) → ${(100 * r.score).toFixed(0)}% (${r.resolved ? 'resolved' : 'unresolved'})`, - failing: r.resolved ? [] : cap.failing(), + failing: r.resolved ? [] : traced.failing(), } const spent: Spend = { iterations: r.completions, tokens: r.tokens, usd: r.usd, ms: r.ms } artifact = { @@ -157,6 +240,7 @@ function surfaceWorkerExecutor( } return artifact }, + traceSource: () => traced.traceSource, teardown: () => Promise.resolve({ destroyed: true }), resultArtifact() { if (!artifact) throw new Error('surfaceWorkerExecutor: resultArtifact before execute') diff --git a/src/runtime/supervise/trace-evidence.ts b/src/runtime/supervise/trace-evidence.ts index 36b6a9f1..fbe92d68 100644 --- a/src/runtime/supervise/trace-evidence.ts +++ b/src/runtime/supervise/trace-evidence.ts @@ -30,6 +30,21 @@ export interface WorkerToolTraceArtifact { readonly spans: ReadonlyArray } +/** Structural guard for the official bounded trace-analysis read contract. */ +export function isTraceAnalysisStore(value: unknown): value is TraceAnalysisStore { + if (value === null || typeof value !== 'object') return false + const store = value as Partial> + return ( + typeof store.getOverview === 'function' && + typeof store.queryTraces === 'function' && + typeof store.countTraces === 'function' && + typeof store.viewTrace === 'function' && + typeof store.viewSpans === 'function' && + typeof store.searchTrace === 'function' && + typeof store.searchSpan === 'function' + ) +} + /** Collect and persist one executor's structured tool trace without changing its task outcome. */ export async function captureWorkerTraceEvidence( readSource: (() => TraceSource | undefined) | undefined, diff --git a/tests/kernel/checks.test.ts b/tests/kernel/checks.test.ts index 099c89b6..b21baefd 100644 --- a/tests/kernel/checks.test.ts +++ b/tests/kernel/checks.test.ts @@ -1,3 +1,4 @@ +import { toolSpansToTraceAnalysisStore } from '@tangle-network/agent-eval' import { describe, expect, it } from 'vitest' import { type Check, @@ -65,6 +66,31 @@ describe('analyst-kind directory', () => { expect(t).toContain('RESULT ok') }) + it('renders bounded TraceAnalysisStore views instead of stringifying the store to an empty object', async () => { + const store = toolSpansToTraceAnalysisStore([ + { + spanId: 'tests-1', + runId: 'surface-worker-1', + kind: 'tool', + name: 'run_tests', + toolName: 'run_tests', + args: {}, + result: '2/3 tests passed. FAILING: test_exact_case', + status: 'ok', + startedAt: 1, + endedAt: 2, + }, + ]) + + const rendered = await renderTrace(store) + + expect(rendered).toContain('OVERVIEW') + expect(rendered).toContain('surface-worker-1') + expect(rendered).toContain('run_tests') + expect(rendered).toContain('test_exact_case') + expect(rendered).not.toBe('{}') + }) + it('runCheck applies the lens via an injected chat → findings', async () => { const chat = async () => '```json\n[{"severity":"medium","claim":"incidents not migrated","evidence_uri":"artifact://db.json","confidence":0.7}]\n```' @@ -79,6 +105,43 @@ describe('analyst-kind directory', () => { expect(out[0]?.evidence_refs[0]?.kind).toBe('artifact') }) + it('passes a bounded store rendering to the MCP check model', async () => { + const store = toolSpansToTraceAnalysisStore([ + { + spanId: 'tests-2', + runId: 'surface-worker-2', + kind: 'tool', + name: 'run_tests', + toolName: 'run_tests', + args: {}, + result: '1/2 tests passed. FAILING: test_rendered', + status: 'ok', + startedAt: 3, + endedAt: 4, + }, + ]) + let userPrompt = '' + + await runCheck( + kind, + store, + { + routerBaseUrl: 'x', + routerKey: 'x', + model: 'm', + chat: async (_system, user) => { + userPrompt = user + return '```json\n[]\n```' + }, + }, + AT, + ) + + expect(userPrompt).toContain('run_tests') + expect(userPrompt).toContain('test_rendered') + expect(userPrompt).not.toContain('WORKER TRACE:\n{}') + }) + it('makeCheckRunner dispatches by kind id; unknown kind → typed error', async () => { const chat = async () => '```json\n[]\n```' const run = makeCheckRunner(defaultChecks, { diff --git a/tests/kernel/supervise-surface.test.ts b/tests/kernel/supervise-surface.test.ts new file mode 100644 index 00000000..195334be --- /dev/null +++ b/tests/kernel/supervise-surface.test.ts @@ -0,0 +1,122 @@ +import type { TraceAnalysisStore } from '@tangle-network/agent-eval' +import { toolSpansToTraceAnalysisStore } from '@tangle-network/agent-eval' +import { describe, expect, it } from 'vitest' +import type { AgenticSurface, AgenticTask } from '../../src/runtime/strategy' +import { failuresAnalyst, traceSurfaceCalls } from '../../src/runtime/supervise-surface' + +const task: AgenticTask = { + id: 'surface-trace-test', + systemPrompt: 'Fix every failing test.', + userPrompt: 'Make the suite pass.', +} + +function fakeSurface(): AgenticSurface { + let testRuns = 0 + return { + name: 'trace-test', + open: async () => ({ id: 'artifact-1', surface: 'trace-test' }), + tools: async () => [], + async call(_handle, name) { + if (name === 'run_tests') { + testRuns += 1 + return testRuns === 1 + ? '2/5 tests passed. FAILING: stale_failure' + : '3/5 tests passed. FAILING: test_alpha, test_beta' + } + if (name === 'explode') throw new Error('tool exploded') + return 'Worker prose claims FAILING: fabricated_from_prose' + }, + score: async () => ({ passes: 3, total: 5, errored: 0 }), + close: async () => undefined, + } +} + +describe('superviseSurface trace evidence', () => { + it('records real surface-call success/error spans and derives exact failures from run_tests only', async () => { + const traced = traceSurfaceCalls(fakeSurface()) + const handle = await traced.surface.open(task) + + await traced.surface.call(handle, 'read_file', { path: 'tests.ts' }) + await traced.surface.call(handle, 'run_tests', {}) + await traced.surface.call(handle, 'run_tests', {}) + await expect(traced.surface.call(handle, 'explode', { reason: 'test' })).rejects.toThrow( + 'tool exploded', + ) + + const spans = await traced.traceSource.collect() + expect( + spans.map((span) => ({ + toolName: span.toolName, + status: span.status, + result: span.result, + })), + ).toEqual([ + { + toolName: 'read_file', + status: 'ok', + result: 'Worker prose claims FAILING: fabricated_from_prose', + }, + { + toolName: 'run_tests', + status: 'ok', + result: '2/5 tests passed. FAILING: stale_failure', + }, + { + toolName: 'run_tests', + status: 'ok', + result: '3/5 tests passed. FAILING: test_alpha, test_beta', + }, + { toolName: 'explode', status: 'error', result: 'ERROR: tool exploded' }, + ]) + + const result = (await failuresAnalyst().run( + 'failures', + toolSpansToTraceAnalysisStore(spans), + )) as { summary: string } + expect(result.summary).toContain('STILL FAILING (2): test_alpha, test_beta') + expect(result.summary).not.toContain('stale_failure') + expect(result.summary).not.toContain('fabricated_from_prose') + }) + + it('refuses to infer failures from the legacy worker prose object', async () => { + const proseOnly = { + resolved: false, + score: 0.4, + shots: 2, + summary: 'STILL FAILING: invented_one', + failing: ['invented_one'], + } + + const result = (await failuresAnalyst().run( + 'failures', + proseOnly as unknown as TraceAnalysisStore, + )) as { summary: string } + + expect(result.summary).toMatch(/Missing structured run_tests span evidence.*Refusing/) + expect(result.summary).not.toContain('invented_one') + }) + + it('treats prose in non-run_tests spans as missing failure evidence', async () => { + const proseOnlyStore = toolSpansToTraceAnalysisStore([ + { + spanId: 'prose-1', + runId: 'surface-worker-prose', + kind: 'tool', + name: 'read_file', + toolName: 'read_file', + args: { path: 'worker-summary.txt' }, + result: 'Worker says FAILING: fabricated_from_span_prose', + status: 'ok', + startedAt: 1, + endedAt: 2, + }, + ]) + + const result = (await failuresAnalyst().run('failures', proseOnlyStore)) as { + summary: string + } + + expect(result.summary).toMatch(/Missing structured run_tests span evidence.*Refusing/) + expect(result.summary).not.toContain('fabricated_from_span_prose') + }) +}) From d1500276ab959c2526c2c0b621715c6038ed4f32 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 07:22:29 -0600 Subject: [PATCH 09/12] fix(supervise): complete full-profile execution proof --- README.md | 8 +- docs/api/agent.md | 81 +- docs/api/candidate-execution.md | 6 + docs/api/index.md | 214 +- docs/api/mcp.md | 165 +- docs/api/primitive-catalog.md | 2 +- docs/api/runtime.md | 1987 ++++++++++++++--- docs/architecture.md | 74 +- examples/ablation-suite/ablation.ts | 56 +- examples/ablation-suite/gepa-driver-prompt.ts | 2 +- examples/supervise/supervise.ts | 18 +- examples/supervisor-loop/run.ts | 10 +- examples/supervisor-loop/shared.ts | 5 +- src/candidate-execution/profile.ts | 1 + src/knowledge/supervised-update.ts | 4 +- src/runtime/supervise/bridge-executor.test.ts | 472 +++- src/runtime/supervise/delegate.ts | 15 +- src/runtime/supervise/dispatch.ts | 8 +- src/runtime/supervise/model-policy.ts | 17 + src/runtime/supervise/supervise.ts | 6 + tests/helpers/resume-driver-child.ts | 2 +- tests/kernel/completion-gate.test.ts | 34 +- tests/kernel/coordination-driver.test.ts | 81 +- tests/kernel/coordination-log.test.ts | 265 +++ tests/kernel/delegate.test.ts | 16 +- .../kernel/driver-inference-metering.test.ts | 171 +- tests/kernel/driver-recursion.test.ts | 27 +- tests/kernel/durable-jsonl.test.ts | 109 + tests/kernel/inbox.test.ts | 68 +- tests/kernel/supervise-convenience.test.ts | 59 + tests/kernel/supervise.test.ts | 121 + tests/kernel/supervisor-authoring.test.ts | 26 +- tests/knowledge-supervised-update.test.ts | 2 +- tests/mcp/delegate.test.ts | 4 +- tests/runtime/bridge-executor.test.ts | 176 +- tests/runtime/executor-profile-model.test.ts | 83 + tests/runtime/mid-flight-steering.test.ts | 127 +- tests/runtime/pi-executor.test.ts | 59 +- tests/runtime/resume-aware-driver.test.ts | 11 +- tests/runtime/spawn-keys.test.ts | 29 +- tests/runtime/stop-rules.test.ts | 10 +- tests/runtime/supervisor-finalizer.test.ts | 6 +- tests/runtime/wait-states.test.ts | 107 +- tests/supervisor-loop-example.test.ts | 6 +- 44 files changed, 4199 insertions(+), 551 deletions(-) create mode 100644 tests/kernel/coordination-log.test.ts create mode 100644 tests/kernel/durable-jsonl.test.ts create mode 100644 tests/runtime/executor-profile-model.test.ts diff --git a/README.md b/README.md index a2b995bf..eb31294d 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,13 @@ One supervisor spawns and steers workers toward a goal. Where the workers run (a import { supervise } from '@tangle-network/agent-runtime/kernel' const result = await supervise( - { name: 'supervisor', harness: null, systemPrompt: 'Delegate to workers; do not solve the task yourself.' }, + { + name: 'supervisor', + harness: 'cli-base', + prompt: { + systemPrompt: 'Delegate to workers; do not solve the task yourself.', + }, + }, 'Implement the feature and make the tests pass.', { budget, router, backend }, // backend = where workers run: router-tools | sandbox+harness | bridge ) diff --git a/docs/api/agent.md b/docs/api/agent.md index 53073cfe..cbf157d7 100644 --- a/docs/api/agent.md +++ b/docs/api/agent.md @@ -1106,16 +1106,73 @@ the loop produces 20 minutes later). AgentProfile axis name, with `custom:` reserved for caller-owned extensions. +*** + +### CanonicalAgentProfileMaterializationAxis + +> **CanonicalAgentProfileMaterializationAxis** = [`KnownAgentProfileMaterializationAxis`](#knownagentprofilematerializationaxis) + +Canonical AgentProfile axes used when checking one complete profile. + ## Variables ### AGENT\_PROFILE\_MATERIALIZATION\_AXES -> `const` **AGENT\_PROFILE\_MATERIALIZATION\_AXES**: readonly \[`"identity"`, `"name"`, `"model"`, `"prompt"`, `"systemPrompt"`, `"instructions"`, `"resources"`, `"files"`, `"resourceInstructions"`, `"skills"`, `"resourceTools"`, `"resourceAgents"`, `"commands"`, `"tools"`, `"permissions"`, `"mcp"`, `"mcpConnections"`, `"connections"`, `"subagents"`, `"hooks"`, `"modes"`, `"confidential"`, `"metadata"`, `"extensions"`\] +> `const` **AGENT\_PROFILE\_MATERIALIZATION\_AXES**: readonly \[`"identity"`, `"name"`, `"description"`, `"version"`, `"tags"`, `"model"`, `"modelDefault"`, `"modelSmall"`, `"modelProvider"`, `"modelReasoningEffort"`, `"modelMetadata"`, `"harness"`, `"prompt"`, `"systemPrompt"`, `"instructions"`, `"resources"`, `"files"`, `"resourceInstructions"`, `"skills"`, `"resourceTools"`, `"resourceAgents"`, `"commands"`, `"resourceFailOnError"`, `"tools"`, `"permissions"`, `"mcp"`, `"mcpConnections"`, `"connections"`, `"subagents"`, `"hooks"`, `"modes"`, `"confidential"`, `"metadata"`, `"extensions"`\] Known AgentProfile axes a run path may or may not carry into execution. *** +### fullProfileMaterialization + +> `const` **fullProfileMaterialization**: [`ProfileMaterializationContract`](#profilematerializationcontract) + +Materialization contract for a run path that executes every canonical AgentProfile axis. + +*** + +### promptModelProfileMaterialization + +> `const` **promptModelProfileMaterialization**: [`ProfileMaterializationContract`](#profilematerializationcontract) + +Materialization contract for an intentionally limited prompt-and-model execution path. +Identity, harness, and metadata are control fields consumed for naming, placement, +authorization, and durable attribution; they are carried without adding worker behavior. +Every behavioral axis other than prompt and model remains unsupported. + +*** + +### worktreeCliProfileMaterialization + +> `const` **worktreeCliProfileMaterialization**: [`ProfileMaterializationContract`](#profilematerializationcontract) + +Materialization contract for a local coding CLI in an isolated git worktree. +The shared workspace materializer carries native tools, permissions, MCP, hooks, subagents, +modes, and file-backed resources when the selected CLI supports their exact values. Runtime +placement concerns (hub connections and confidential execution), provider-native extensions, +unused model hints, and `resources.failOnError` are deliberately absent so they fail before a +worktree or executor is created rather than being mistaken for an effective candidate change. + +*** + +### controlProfileMaterialization + +> `const` **controlProfileMaterialization**: [`ProfileMaterializationContract`](#profilematerializationcontract) + +Materialization contract for a raw process path that carries only control/identity fields. + +*** + +### promptControlProfileMaterialization + +> `const` **promptControlProfileMaterialization**: [`ProfileMaterializationContract`](#profilematerializationcontract) + +Materialization contract for an injected inference function whose surrounding driver still +applies the profile prompt, name, placement, and metadata, but not model selection. + +*** + ### sandboxActProfileMaterialization > `const` **sandboxActProfileMaterialization**: [`ProfileMaterializationContract`](#profilematerializationcontract) @@ -1275,6 +1332,28 @@ Define the profile axes a concrete run path actually carries into execution. *** +### profileMaterializationAxes() + +> **profileMaterializationAxes**(`profile`): readonly (`"metadata"` \| `"resources"` \| `"name"` \| `"tools"` \| `"model"` \| `"mcp"` \| `"connections"` \| `"subagents"` \| `"hooks"` \| `"modes"` \| `"extensions"` \| `"description"` \| `"version"` \| `"tags"` \| `"prompt"` \| `"harness"` \| `"permissions"` \| `"confidential"` \| `"systemPrompt"` \| `"instructions"` \| `"skills"` \| `"identity"` \| `"modelDefault"` \| `"modelSmall"` \| `"modelProvider"` \| `"modelReasoningEffort"` \| `"modelMetadata"` \| `"files"` \| `"resourceInstructions"` \| `"resourceTools"` \| `"resourceAgents"` \| `"commands"` \| `"resourceFailOnError"` \| `"mcpConnections"`)[] + +Return the exact canonical axes a complete profile actually requests. Compound prompt, model, +identity, and resource objects are split so a path cannot claim an entire object while silently +dropping one of its fields. +Empty strings, arrays, and nested records do not claim support; explicit +scalar values such as `false` and `0` remain meaningful requests. + +#### Parameters + +##### profile + +`AgentProfile` + +#### Returns + +readonly (`"metadata"` \| `"resources"` \| `"name"` \| `"tools"` \| `"model"` \| `"mcp"` \| `"connections"` \| `"subagents"` \| `"hooks"` \| `"modes"` \| `"extensions"` \| `"description"` \| `"version"` \| `"tags"` \| `"prompt"` \| `"harness"` \| `"permissions"` \| `"confidential"` \| `"systemPrompt"` \| `"instructions"` \| `"skills"` \| `"identity"` \| `"modelDefault"` \| `"modelSmall"` \| `"modelProvider"` \| `"modelReasoningEffort"` \| `"modelMetadata"` \| `"files"` \| `"resourceInstructions"` \| `"resourceTools"` \| `"resourceAgents"` \| `"commands"` \| `"resourceFailOnError"` \| `"mcpConnections"`)[] + +*** + ### validateProfileMaterialization() > **validateProfileMaterialization**(`options`): readonly [`ProfileMaterializationIssue`](#profilematerializationissue)[] diff --git a/docs/api/candidate-execution.md b/docs/api/candidate-execution.md index b1d14ab1..ff931393 100644 --- a/docs/api/candidate-execution.md +++ b/docs/api/candidate-execution.md @@ -254,6 +254,12 @@ Re-exports [prepareAgentCandidateExecution](index.md#prepareagentcandidateexecut *** +### agentCandidateProfileAsAgentProfile + +Re-exports [agentCandidateProfileAsAgentProfile](index.md#agentcandidateprofileasagentprofile) + +*** + ### applyExactAgentProfileDiff Re-exports [applyExactAgentProfileDiff](index.md#applyexactagentprofilediff) diff --git a/docs/api/index.md b/docs/api/index.md index ab5bbbdd..88b6c2e0 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -5967,7 +5967,7 @@ Findings to fall back to when the generation had NO failing cells, so a ###### profile -[`SupervisorProfile`](runtime.md#supervisorprofile) +`AgentProfile` ###### task @@ -6293,7 +6293,7 @@ Findings to fall back to when the generation had NO failing cells, so a ###### profile -[`SupervisorProfile`](runtime.md#supervisorprofile) +`AgentProfile` ###### task @@ -6656,7 +6656,7 @@ Max research rounds (correct-on-veto remediation). Default 1. ###### trace -`unknown` +`TraceAnalysisStore` ###### Returns @@ -7952,7 +7952,7 @@ What a finalizer gets to decide with. `delivered` is the ONLY output material; ` ##### budget -> `readonly` **budget**: `Readonly`\<\{ `tokensLeft`: `number`; `usdLeft`: `number`; `usdCapped`: `boolean`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +> `readonly` **budget**: `Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> *** @@ -7968,7 +7968,7 @@ meters every runtime identically. Built-in implementations (in `runtime.ts`, NOT variants here): router/inline (a direct Router/HTTP inference call, no box), sandbox (COMPOSES `runAgentRounds` as a leaf, forwarding PR #150's optional `lineage` passthrough — does NOT reinvent checkpoint/fork), cli -(Halo/RLM subprocess; `budgetExempt`, excluded from equal-k by construction). A user's +(Halo/RLM subprocess; `budgetExempt`, refused by budgeted supervision). A user's own agent (mastra/agno/raw HTTP/anything) is first-class by implementing this interface. #### Type Parameters @@ -7989,9 +7989,10 @@ Stable runtime tag for traces + the equal-k exemption check. > `readonly` `optional` **budgetExempt?**: `boolean` -When true, this executor's spend is NOT metered against the conserved pool and its -iterations are excluded from the equal-k assertion (a `cli` subprocess without -token accounting). Fail-loud everywhere else: a metered executor MUST report usage. +When true, this executor cannot report the usage a conserved pool would need (for example, a +subscription CLI with no token receipt). `Executor` can still be used directly, but `Scope` +refuses it before `execute` so unknown compute can never appear as measured zero in a +supervised or equal-resource run. A metered executor MUST report usage. #### Methods @@ -8114,6 +8115,21 @@ driver branched on, its verdict, and the conserved spend. Read once, after settl > **spent**: [`Spend`](#spend) +##### accounting()? + +> `optional` **accounting**(): [`ExecutorAccounting`](runtime.md#executoraccounting) \| `undefined` + +Optional accounting split for recursive executors. +`reported` is the child-work spend written on this node's settlement; `reservation` is the +whole amount reconciled against this node's parent reservation. +They differ when a driver owns a nested allocation: its child work and own inference consume +that allocation together, while the journal keeps those two categories separate. +Valid after `execute` resolves or throws; ordinary leaf executors omit it. + +###### Returns + +[`ExecutorAccounting`](runtime.md#executoraccounting) \| `undefined` + ##### metered()? > `optional` **metered**(): [`Spend`](#spend) \| `undefined` @@ -8133,11 +8149,12 @@ executors omit it (returns `undefined`). ### AgentSpec -`AgentProfile` does NOT carry a `harness`/backend field — `harness` lives on the -sandbox SDK's `BackendConfig`, not the portable profile. So an agent is mapped to its -executor through this MINIMAL wrapper, never by fabricating a field onto `AgentProfile`. +`AgentProfile.harness` is a portable preference; this wrapper records the executor decision for +one concrete run. A caller may honor the preference, override it for a comparison cell, or supply +an executor directly, without changing the profile's behavioral identity. Resolution (in `runtime.ts`): + - `executorFactory` present → BYO: build it after admission with the live context. - `executor` present → BYO: use it verbatim (a user's own `Executor`). - `harness === null` → router/inline: a direct Router call, no box. - `harness` is a `BackendType` → sandbox: compose `runAgentRounds` against `profile` on that backend. @@ -8155,6 +8172,20 @@ Fail loud on an unresolvable spec (no executor and an unknown harness). `null` selects router/inline; a `BackendType` selects the sandboxed harness. +##### execution? + +> `readonly` `optional` **execution?**: [`AgentExecutionRef`](runtime.md#agentexecutionref) + +Trusted candidate/campaign attribution supplied by the caller. Profile/task digests are + computed by Scope from the exact values it executes and cannot be supplied here. + +##### executorFactory? + +> `readonly` `optional` **executorFactory?**: [`ExecutorFactory`](runtime.md#executorfactory)\<`unknown`\> + +Per-spawn factory carrying caller configuration. Constructed only after admission, with the + real child signal and nested-scope context. + ##### executor? > `readonly` `optional` **executor?**: [`Executor`](#executor-2)\<`unknown`\> @@ -8202,8 +8233,8 @@ Register a factory for a named runtime. Throws on a duplicate name (fail loud). > **resolve**\<`Out`\>(`spec`): \{ `succeeded`: `true`; `value`: [`ExecutorFactory`](runtime.md#executorfactory)\<`Out`\>; \} \| \{ `succeeded`: `false`; `error`: `string`; \} -Resolve a spec to a factory. Precedence: a BYO `spec.executor` → a trivial factory -returning it; else `harness === null` → the `'router'` factory; else a registered +Resolve a spec to a factory. Precedence: a BYO `spec.executorFactory` → `spec.executor` → +`harness === null` → the `'router'` factory; else a registered factory for the harness-derived runtime. Returns a typed outcome — the caller inspects `succeeded` before `value` (no silent fallback). @@ -8264,6 +8295,13 @@ Conserved spend, reconciled from the normalized `UsageEvent` stream. Tokens and > **tokens**: [`LoopTokenUsage`](runtime.md#looptokenusage) +##### tokensKnown? + +> `optional` **tokensKnown?**: `boolean` + +Token accounting is known unless explicitly false. Missing provider usage must never be + interpreted as zero under a token ceiling. + ##### usdKnown? > `optional` **usdKnown?**: `boolean` @@ -8325,21 +8363,32 @@ The live tree — reads the in-memory nursery, not the journal. ##### budget -> `readonly` **budget**: `Readonly`\<\{ `tokensLeft`: `number`; `usdLeft`: `number`; `usdCapped`: `boolean`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +> `readonly` **budget**: `Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> Conserved-pool readouts (post-reservation). +##### workerCapacity + +> `readonly` **workerCapacity**: `Readonly`\<\{ `live`: `number`; `freeSlots`: `number` \| `null`; \}\> + +One tree-wide view of simultaneous spawned work. Every nested scope reads the same counter; + the root agent itself is not a spawned worker. `freeSlots` is `null` when no limit is set. + #### Methods ##### spawn() > **spawn**\<`C`\>(`agent`, `task`, `opts`): \{ `ok`: `true`; `handle`: [`Handle`](runtime.md#handle-2)\<`C`\>; `prior?`: [`SpawnPrior`](runtime.md#spawnprior)\<`C`\>; \} \| \{ `ok`: `false`; `reason`: [`SpawnRejection`](runtime.md#spawnrejection); \} -Spawn a child. Reserves `opts.budget` from the conserved pool atomically; refunds the -unspent remainder on settle. Returns a typed outcome — fail-closed on an exhausted -pool, an exceeded depth ceiling, or a still-live duplicate `key` (the caller inspects -`ok` before `handle`). A KEYED spawn whose key already settled `done` spends nothing: -it returns the committed result on `prior` instead of re-running (see `SpawnOpts.key`). +Spawn a child. For a fresh key or an unkeyed spawn, tree-wide worker admission happens before a +lazy factory is called, so a full worker allocation creates no worker, executor, or reservation. +Reserves `opts.budget` from the conserved pool atomically; refunds the unspent remainder on +settle. Returns a typed outcome — fail-closed on an exhausted pool, an exceeded depth ceiling, a +full worker allocation, or a still-live duplicate `key` (the caller inspects `ok` before +`handle`). A KEYED spawn whose key already settled `done` invokes the factory only far enough to +prepare and authorize the exact profile/task identity, then compares that identity with the +journal. On a match it spends nothing, constructs no executor, reserves no budget, and runs no +work: it returns the committed result on `prior` (see `SpawnOpts.key`). ###### Type Parameters @@ -8351,7 +8400,7 @@ it returns the committed result on `prior` instead of re-running (see `SpawnOpts ###### agent -[`Agent`](runtime.md#agent-1)\<`unknown`, `C`\> +[`Agent`](runtime.md#agent-1)\<`unknown`, `C`\> \| (() => [`Agent`](runtime.md#agent-1)\<`unknown`, `C`\>) ###### task @@ -11105,11 +11154,12 @@ Mode → configured runner. Partial: only register the modes a ### CoordinationEvent -> **CoordinationEvent** = \{ `type`: `"question"`; `question`: [`QuestionRecord`](mcp.md#questionrecord); \} \| \{ `type`: `"settled"`; `worker`: [`SettledWorker`](mcp.md#settledworker); \} \| \{ `type`: `"finding"`; `finding`: [`AnalystFindingEvent`](runtime.md#analystfindingevent); \} \| \{ `type`: `"steer"`; `down`: [`DownMessageEvent`](runtime.md#downmessageevent); \} \| \{ `type`: `"answer"`; `down`: [`DownMessageEvent`](runtime.md#downmessageevent); `questionId`: `string`; \} +> **CoordinationEvent** = \{ `type`: `"question"`; `question`: [`QuestionRecord`](mcp.md#questionrecord); \} \| \{ `type`: `"settled"`; `worker`: [`SettledWorker`](mcp.md#settledworker); \} \| \{ `type`: `"finding"`; `finding`: [`AnalystFindingEvent`](runtime.md#analystfindingevent); \} \| \{ `type`: `"steer"`; `down`: [`DownMessageEvent`](runtime.md#downmessageevent); \} \| \{ `type`: `"answer"`; `down`: [`DownMessageEvent`](runtime.md#downmessageevent); `questionId`: `string`; \} \| \{ `type`: `"instruction"`; `instruction`: [`ContinuationInstruction`](runtime.md#continuationinstruction); \} \| \{ `type`: `"delivery-attempt"`; `attempt`: [`DownMessageDeliveryAttempt`](runtime.md#downmessagedeliveryattempt); \} Every message on the one typed pipe. UP (child→parent): question / settled / finding — queued for - the driver to `pull`. DOWN (parent→child): steer / answer — record-only (history + subscribers), - routed to the child inbox. New kinds are additive. + the driver to `pull`. An `instruction` is the pre-delivery authorization receipt and is retained + as evidence. DOWN (parent→child): steer / answer — record-only (history + subscribers), routed + to the child inbox. Receipts are never auto-delivered on restart. New kinds are additive. *** @@ -11209,9 +11259,51 @@ The finalization seam: ledger in, output (or `undefined` = nothing deliverable) *** +### WorkerTraceUnavailableReason + +> **WorkerTraceUnavailableReason** = `"execution-did-not-start"` \| `"executor-did-not-expose-trace-source"` \| `"trace-source-unavailable"` \| `"no-tool-spans-captured"` \| `"invalid-tool-spans"` \| `"trace-collection-failed"` \| `"trace-persistence-failed"` \| `"legacy-settlement-without-trace-evidence"` \| `"not-an-executor"` + +Why Runtime cannot provide structured tool-call evidence for one settled execution. + +*** + +### WorkerTraceEvidence + +> **WorkerTraceEvidence** = \{ `status`: `"available"`; `traceRef`: `string`; `spanCount`: `number`; \} \| \{ `status`: `"unavailable"`; `reason`: [`WorkerTraceUnavailableReason`](#workertraceunavailablereason); \} + +Durable proof of a worker's structured tool trace, or the exact reason it is unavailable. + +#### Union Members + +##### Type Literal + +\{ `status`: `"available"`; `traceRef`: `string`; `spanCount`: `number`; \} + +###### status + +> `readonly` **status**: `"available"` + +###### traceRef + +> `readonly` **traceRef**: `string` + +Content-addressed pointer to a persisted `WorkerToolTraceArtifact`. + +###### spanCount + +> `readonly` **spanCount**: `number` + +*** + +##### Type Literal + +\{ `status`: `"unavailable"`; `reason`: [`WorkerTraceUnavailableReason`](#workertraceunavailablereason); \} + +*** + ### Settled -> **Settled**\<`Out`\> = \{ `kind`: `"done"`; `handle`: [`Handle`](runtime.md#handle-2)\<`Out`\>; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `seq`: `number`; \} \| \{ `kind`: `"down"`; `handle`: [`Handle`](runtime.md#handle-2)\<`Out`\>; `reason`: `string`; `infra`: `boolean`; `restartCount`: `number`; `seq`: `number`; \} +> **Settled**\<`Out`\> = \{ `kind`: `"done"`; `handle`: [`Handle`](runtime.md#handle-2)\<`Out`\>; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `settledAt?`: `number`; `seq`: `number`; \} \| \{ `kind`: `"down"`; `handle`: [`Handle`](runtime.md#handle-2)\<`Out`\>; `reason`: `string`; `infra`: `boolean`; `restartCount`: `number`; `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `settledAt?`: `number`; `seq`: `number`; \} A settled child, delivered by `scope.next()`. `seq` is the monotonic cursor order `next()` yielded this settlement (B2) — NOT wall-clock — and replay delivers strictly @@ -11227,13 +11319,53 @@ in `seq` order. `outRef` rehydrates `out` from the `ResultBlobStore` on replay. ##### Type Literal -\{ `kind`: `"done"`; `handle`: [`Handle`](runtime.md#handle-2)\<`Out`\>; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `seq`: `number`; \} +\{ `kind`: `"done"`; `handle`: [`Handle`](runtime.md#handle-2)\<`Out`\>; `out`: `Out`; `outRef`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](#spend); `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `settledAt?`: `number`; `seq`: `number`; \} + +###### kind + +> **kind**: `"done"` + +###### handle + +> **handle**: [`Handle`](runtime.md#handle-2)\<`Out`\> + +###### out + +> **out**: `Out` + +###### outRef + +> **outRef**: `string` + +###### verdict? + +> `optional` **verdict?**: `DefaultVerdict` + +###### spent + +> **spent**: [`Spend`](#spend) + +###### trace + +> **trace**: [`WorkerTraceEvidence`](#workertraceevidence) + +Structured tool evidence captured before this settlement was journaled. + +###### settledAt? + +> `optional` **settledAt?**: `number` + +Epoch ms parsed from the durable settlement record when available. + +###### seq + +> **seq**: `number` *** ##### Type Literal -\{ `kind`: `"down"`; `handle`: [`Handle`](runtime.md#handle-2)\<`Out`\>; `reason`: `string`; `infra`: `boolean`; `restartCount`: `number`; `seq`: `number`; \} +\{ `kind`: `"down"`; `handle`: [`Handle`](runtime.md#handle-2)\<`Out`\>; `reason`: `string`; `infra`: `boolean`; `restartCount`: `number`; `trace`: [`WorkerTraceEvidence`](#workertraceevidence); `settledAt?`: `number`; `seq`: `number`; \} ###### kind @@ -11257,6 +11389,18 @@ True = infrastructure failure (excluded from merge `n` / equal-k), not a bad res > **restartCount**: `number` +###### trace + +> **trace**: [`WorkerTraceEvidence`](#workertraceevidence) + +Partial structured tool evidence captured before this failure was journaled. + +###### settledAt? + +> `optional` **settledAt?**: `number` + +Epoch ms parsed from the durable settlement/cancellation record when available. + ###### seq > **seq**: `number` @@ -12439,6 +12583,24 @@ Apply one exact diff and reject any value that cannot be preserved canonically. *** +### agentCandidateProfileAsAgentProfile() + +> **agentCandidateProfileAsAgentProfile**(`candidate`): `AgentProfile` + +Convert the candidate profile contract into the portable interface profile it represents. + +#### Parameters + +##### candidate + +`AgentCandidateProfile` + +#### Returns + +`AgentProfile` + +*** + ### createProtectedAgentCandidateModelPort() > **createProtectedAgentCandidateModelPort**(`options`): [`AgentCandidateModelPort`](#agentcandidatemodelport) diff --git a/docs/api/mcp.md b/docs/api/mcp.md index b7d05041..a16d38c6 100644 --- a/docs/api/mcp.md +++ b/docs/api/mcp.md @@ -2134,7 +2134,7 @@ Which harness handled this delegation. ###### Inherited from -[`LoopSandboxPlacement`](runtime.md#loopsandboxplacement).[`kind`](runtime.md#kind-6) +[`LoopSandboxPlacement`](runtime.md#loopsandboxplacement).[`kind`](runtime.md#kind-10) ##### sandboxId? @@ -3897,6 +3897,36 @@ A worker the driver has drained via `await_event`. > `readonly` **status**: `"done"` \| `"down"` +##### assignmentId? + +> `readonly` `optional` **assignmentId?**: `string` + +Stable manager-scoped assignment, including deterministic unkeyed siblings. + +##### identity? + +> `readonly` `optional` **identity?**: [`NodeExecutionIdentity`](runtime.md#nodeexecutionidentity) + +Exact profile/task/candidate identity authorized for this node. + +##### materialization? + +> `readonly` `optional` **materialization?**: [`ProfileMaterializationReceipt`](runtime.md#profilematerializationreceipt) + +Stable effective execution plan, or an explicit unknown receipt. + +##### executionBindings? + +> `readonly` `optional` **executionBindings?**: readonly [`ExecutionBindingReceipt`](runtime.md#executionbindingreceipt)[] + +Backend bindings for each attempt, in durable oldest-first order. + +##### spent? + +> `readonly` `optional` **spent?**: [`Spend`](index.md#spend) + +Conserved spend. Missing means unavailable; unknown accounting remains explicitly unknown. + ##### score? > `readonly` `optional` **score?**: `number` @@ -3913,13 +3943,24 @@ A worker the driver has drained via `await_event`. > `readonly` `optional` **reason?**: `string` +##### trace + +> `readonly` **trace**: [`WorkerTraceEvidence`](index.md#workertraceevidence) + +Structured tool-call evidence, never the worker's final prose. + +##### resumed? + +> `readonly` `optional` **resumed?**: `boolean` + +True when projected from a prior process of the same durable run. + ##### settledAt? > `readonly` `optional` **settledAt?**: `number` -Epoch ms the ledger recorded this settlement — the resolution a progress-based stop rule - needs to answer "how long since anything landed?" without inventing a timestamp at read - time. Stamped when the cursor yields the settlement, not when a reader first looks. +Epoch ms from the durable terminal record — the resolution a progress-based stop rule needs + to answer "how long since anything landed?" without inventing a timestamp at read time. *** @@ -4041,7 +4082,7 @@ Epoch ms the ledger recorded this settlement — the resolution a progress-based ##### status -> `readonly` **status**: `"open"` \| `"answered"` \| `"deferred"` \| `"escalated"` +> `readonly` **status**: `"deferred"` \| `"open"` \| `"answered"` \| `"escalated"` ##### decision? @@ -4079,7 +4120,9 @@ Epoch ms the ledger recorded this settlement — the resolution a progress-based ##### onEvent? -> `readonly` `optional` **onEvent?**: (`event`) => `void` \| `Promise`\<`void`\> +> `readonly` `optional` **onEvent?**: (`event`, `record`) => `void` \| `Promise`\<`void`\> + +Event-first for source compatibility; the second argument is its exact bus ordering stamp. ###### Parameters @@ -4087,10 +4130,28 @@ Epoch ms the ledger recorded this settlement — the resolution a progress-based [`CoordinationEvent`](index.md#coordinationevent) +###### record + +[`BusRecord`](runtime.md#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> + ###### Returns `void` \| `Promise`\<`void`\> +##### replaySettlements? + +> `readonly` `optional` **replaySettlements?**: `boolean` + +Re-publish resumed settlements through the awaited observer before the driver starts. This is + the crash-window recovery path for product transactions; off preserves low-level legacy reads. + +##### authorizeDownMessage? + +> `readonly` `optional` **authorizeDownMessage?**: [`AuthorizeDownMessage`](runtime.md#authorizedownmessage) + +Authorize each continuation against the exact worker identity. The returned instruction is +detached, recorded durably through `onEvent`, and only then delivered. + ##### questionPolicy? > `readonly` `optional` **questionPolicy?**: [`QuestionPolicy`](#questionpolicy) @@ -4112,7 +4173,8 @@ Hard cap on how many workers may be LIVE (spawned but not yet settled) at once. counts the scope's non-terminal nodes and fails closed (`error: 'max-live-workers'`) BEFORE reserving from the pool when the cap is already met — a concurrency fence on top of the conserved-budget fence (the pool bounds total work; this bounds simultaneous work, e.g. live - sandboxes/boxes). Omit or `<= 0` = no cap (the prior behavior; the pool stays the only fence). + sandboxes/boxes). A tree-wide limit owned by `Scope` takes precedence when present; this field + is the local form for a caller-owned scope. Omit or `<= 0` = no local cap. ##### awaitTimeoutMs? @@ -4199,6 +4261,16 @@ choice, steerable counterpart to the one-shot own-sandbox delegation MCP. #### Methods +##### ready() + +> **ready**(): `Promise`\<`void`\> + +Commit any resume-time event replay before a supervisor can reason or an MCP can listen. + +###### Returns + +`Promise`\<`void`\> + ##### isStopped() > **isStopped**(): `boolean` @@ -4235,9 +4307,9 @@ readonly [`QuestionRecord`](#questionrecord)[] > **history**(): readonly [`BusRecord`](runtime.md#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\>[] -The full ordered log of every bus event — UP (settled / question / finding) and DOWN - (steer / answer) — the observability audit + replay trail. Each record carries seq, - timestamp, and priority. +The full ordered log of every bus event — UP (settled / question / finding), authorized + instruction receipts, and DOWN delivery outcomes (steer / answer). Each record carries seq, + timestamp, and priority. A receipt is evidence and is never auto-delivered on restart. ###### Returns @@ -7304,18 +7376,39 @@ Lift validated raw rows into `AnalystFinding`s (agent-eval `makeFinding` stamps ### renderTrace() +#### Call Signature + +> **renderTrace**(`trace`): `Promise`\<`string`\> + +Render a worker's trace (tool calls + results) into the text an analyst lens reads. Conversation + values remain synchronous; the official bounded store is read asynchronously through its own + overview and view operations instead of stringifying its prototype to `{}`. + +##### Parameters + +###### trace + +`TraceAnalysisStore` + +##### Returns + +`Promise`\<`string`\> + +#### Call Signature + > **renderTrace**(`trace`): `string` -Render a worker's trace (tool calls + results) into the text an analyst lens reads. Generic over - the trace shape: a `{ messages }` conversation, a bare message array, else stringified. +Render a worker's trace (tool calls + results) into the text an analyst lens reads. Conversation + values remain synchronous; the official bounded store is read asynchronously through its own + overview and view operations instead of stringifying its prototype to `{}`. -#### Parameters +##### Parameters -##### trace +###### trace `unknown` -#### Returns +##### Returns `string` @@ -7745,6 +7838,24 @@ Re-exports [AnalystRegistry](index.md#analystregistry) *** +### AuthorizeDownMessage + +Re-exports [AuthorizeDownMessage](runtime.md#authorizedownmessage) + +*** + +### AuthorizedDownMessage + +Re-exports [AuthorizedDownMessage](runtime.md#authorizeddownmessage) + +*** + +### ContinuationInstruction + +Re-exports [ContinuationInstruction](runtime.md#continuationinstruction) + +*** + ### CoordinationEvent Re-exports [CoordinationEvent](index.md#coordinationevent) @@ -7757,6 +7868,24 @@ Re-exports [DEFAULT_AWAIT_EVENT_TIMEOUT_MS](runtime.md#default_await_event_timeo *** +### DownMessageAuthorizationInput + +Re-exports [DownMessageAuthorizationInput](runtime.md#downmessageauthorizationinput) + +*** + +### DownMessageDeliveryAttempt + +Re-exports [DownMessageDeliveryAttempt](runtime.md#downmessagedeliveryattempt) + +*** + +### DownMessageDeliveryOutcome + +Re-exports [DownMessageDeliveryOutcome](runtime.md#downmessagedeliveryoutcome) + +*** + ### DownMessageEvent Re-exports [DownMessageEvent](runtime.md#downmessageevent) @@ -7766,3 +7895,9 @@ Re-exports [DownMessageEvent](runtime.md#downmessageevent) ### MakeWorkerAgent Re-exports [MakeWorkerAgent](runtime.md#makeworkeragent) + +*** + +### WorkerSpawnContext + +Re-exports [WorkerSpawnContext](runtime.md#workerspawncontext) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 5c40edd1..2b8a3b90 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -1264,7 +1264,7 @@ Import from `@tangle-network/agent-runtime/mcp` — 213 exports. | `readMemoryItemsFile` | function | Read a memory store file: a JSON array, or JSONL (one `MemoryItem` per line). | | `readTraceContextFromEnv` | function | Read trace context from the process environment. | | `removeWorktree` | function | Remove a git worktree and delete its branch. Already-removed paths are harmless; every other | -| `renderTrace` | function | Render a worker's trace (tool calls + results) into the text an analyst lens reads. Generic over | +| `renderTrace` | function | Render a worker's trace (tool calls + results) into the text an analyst lens reads. Conversation | | `resolveMemoryFromEnv` | function | Resolve the bin's memory from `AGENT_MEMORY_FILE` (durable store) and/or | | `runCheck` | function | Run ONE lens over a trace → findings. Generic over any kind: prompt = the lens + the agent-eval | | `runDetachedTurn` | function | Dispatch one detached turn and advance it to a terminal state with | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index a20caaeb..1106d0df 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -619,7 +619,7 @@ FS-backed `CoordinationLog`: append-only JSONL, fsynced per record. ##### append() -> **append**(`runId`, `event`, `at`): `Promise`\<`void`\> +> **append**(`runId`, `record`, `ownerId?`): `Promise`\<`void`\> ###### Parameters @@ -627,11 +627,11 @@ FS-backed `CoordinationLog`: append-only JSONL, fsynced per record. `string` -###### event +###### record -[`CoordinationEvent`](index.md#coordinationevent) +[`BusRecord`](#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> -###### at +###### ownerId? `string` @@ -645,7 +645,7 @@ FS-backed `CoordinationLog`: append-only JSONL, fsynced per record. ##### load() -> **load**(`runId`): `Promise`\<[`PriorCoordination`](#priorcoordination-1)\> +> **load**(`runId`, `ownerId?`): `Promise`\<[`PriorCoordination`](#priorcoordination-1)\> ###### Parameters @@ -653,6 +653,10 @@ FS-backed `CoordinationLog`: append-only JSONL, fsynced per record. `string` +###### ownerId? + +`string` + ###### Returns `Promise`\<[`PriorCoordination`](#priorcoordination-1)\> @@ -683,13 +687,52 @@ A trace-analyst result re-entered as a message on the bus (the `finding` event k *** +### DownMessageDeliveryAttempt + +A durable marker written after authorization and immediately before Runtime calls `Scope.send`. +If a process dies with this marker but no matching outcome, delivery is unknown and is never +replayed automatically. + +#### Properties + +##### receiptId + +> `readonly` **receiptId**: `string` + +##### kind + +> `readonly` **kind**: `"steer"` \| `"answer"` + +##### toWorker + +> `readonly` **toWorker**: `string` + +##### instructionDigest + +> `readonly` **instructionDigest**: `string` + +##### interrupt + +> `readonly` **interrupt**: `boolean` + +##### questionId? + +> `readonly` `optional` **questionId?**: `string` + +*** + ### DownMessageEvent -A parent→child message (the down-leg): recorded for observability, delivered via the child inbox, - never pulled back by the parent. `delivered` mirrors whether the live child accepted it. +A parent→child delivery result (the down-leg): recorded for observability, never pulled back by +the parent. `receiptId` and `instructionDigest` link it to the pre-delivery authorization receipt +and attempt marker. #### Properties +##### receiptId + +> `readonly` **receiptId**: `string` + ##### toWorker > `readonly` **toWorker**: `string` @@ -698,10 +741,161 @@ A parent→child message (the down-leg): recorded for observability, delivered v > `readonly` **instruction**: `string` +##### instructionDigest + +> `readonly` **instructionDigest**: `string` + ##### delivered > `readonly` **delivered**: `boolean` +##### outcome + +> `readonly` **outcome**: [`DownMessageDeliveryOutcome`](#downmessagedeliveryoutcome) + +##### error? + +> `readonly` `optional` **error?**: `string` + +*** + +### ContinuationInstruction + +Durable authorization receipt written before a continuation reaches a worker. + +#### Properties + +##### receiptId + +> `readonly` **receiptId**: `string` + +##### kind + +> `readonly` **kind**: `"steer"` \| `"answer"` + +##### toWorker + +> `readonly` **toWorker**: `string` + +##### instruction + +> `readonly` **instruction**: `string` + +##### instructionDigest + +> `readonly` **instructionDigest**: `string` + +##### workerIdentity? + +> `readonly` `optional` **workerIdentity?**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +##### interrupt + +> `readonly` **interrupt**: `boolean` + +##### questionId? + +> `readonly` `optional` **questionId?**: `string` + +*** + +### DownMessageAuthorizationInput + +Detached continuation bytes and exact worker identity presented to product authorization before +Runtime records or delivers a steer/answer. + +#### Properties + +##### kind + +> `readonly` **kind**: `"steer"` \| `"answer"` + +##### workerId + +> `readonly` **workerId**: `string` + +##### workerIdentity + +> `readonly` **workerIdentity**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +##### instruction + +> `readonly` **instruction**: `string` + +##### interrupt + +> `readonly` **interrupt**: `boolean` + +##### questionId? + +> `readonly` `optional` **questionId?**: `string` + +*** + +### AuthorizedDownMessage + +Product-authorized continuation bytes. Returning a narrowed instruction replaces the proposed +bytes; throwing refuses delivery. + +#### Properties + +##### instruction + +> `readonly` **instruction**: `string` + +*** + +### WorkerSpawnContext + +Immutable task, allocation, identity attribution, and semantic key supplied while a manager's +complete worker profile is prepared for one spawn. + +#### Properties + +##### assignmentId + +> `readonly` **assignmentId**: `string` + +Stable assignment identity within this manager. A semantic key wins; otherwise Runtime mints +the manager's deterministic pre-factory spawn ordinal so identical unkeyed siblings stay +isolated and can recover by issuing the same assignments in the same order. + +##### parentNodeId + +> `readonly` **parentNodeId**: `string` + +Trusted concrete manager node authorizing this spawn. Never accepted from model arguments. + +##### budget + +> `readonly` **budget**: [`Budget`](index.md#budget-4) + +The exact allocation this node receives after the tool's optional override is merged. + +##### task + +> `readonly` **task**: `unknown` + +Detached, deeply immutable task bytes from this spawn request. + +##### label + +> `readonly` **label**: `string` + +Exact trace label selected for this spawn. + +##### key? + +> `readonly` `optional` **key?**: `string` + +Semantic restart key, when the manager supplied one. + +##### execution? + +> `readonly` `optional` **execution?**: [`AgentExecutionRef`](#agentexecutionref) + +Trusted candidate/campaign attribution attached by product authorization. + *** ### WorktreeCommandResult @@ -3267,7 +3461,7 @@ The scope analyst (selector≠judge firewall) the combinator steers from. Absent ##### spawnChild() -> **spawnChild**(`name`, `spec`): [`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-1)\<`D`\>\> +> **spawnChild**(`name`, `spec`): [`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-2)\<`D`\>\> Wrap an `AgentSpec` into a leaf `Agent` carrying it as `executorSpec`, so the shape can `scope.spawn(spawnChild(spec), task, opts)`. `name` labels the child for traces. The @@ -3286,7 +3480,7 @@ spec drives the resolved `Executor`; `act` exists only to satisfy the `Agent` sh ###### Returns -[`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-1)\<`D`\>\> +[`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-2)\<`D`\>\> ##### childSpec() @@ -3462,7 +3656,7 @@ OTP intensity breaker bounds, forwarded to the supervisor verbatim. ##### handle? -> `readonly` `optional` **handle?**: [`RootHandle`](#roothandle)\<[`Outcome`](#outcome-1)\<`D`\>\> +> `readonly` `optional` **handle?**: [`RootHandle`](#roothandle)\<[`Outcome`](#outcome-2)\<`D`\>\> A live root handle to attach (view/signal/abort) before the run starts. @@ -3557,7 +3751,7 @@ Derive this stage's task from the prior stage's deliverable (or the root task fo ##### collect() -> **collect**(`settled`): [`Outcome`](#outcome-1)\<`StepOut`\> +> **collect**(`settled`): [`Outcome`](#outcome-2)\<`StepOut`\> Read this stage's settled child output into the typed `StepOut` the next stage feeds on. Fail loud (return a `blocked`) when the child produced nothing usable for the next stage. @@ -3566,11 +3760,11 @@ Read this stage's settled child output into the typed `StepOut` the next stage f ###### settled -[`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`StepOut`\>\> +[`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`StepOut`\>\> ###### Returns -[`Outcome`](#outcome-1)\<`StepOut`\> +[`Outcome`](#outcome-2)\<`StepOut`\> *** @@ -3730,7 +3924,7 @@ How a fanout's synthesis child is built + read. `synthesisTask` projects the dra ###### gathered -readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`D`\>\>[] +readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`D`\>\>[] ###### ctx @@ -3742,17 +3936,17 @@ readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`D`\>\>[] ##### collect() -> **collect**(`settled`): [`Outcome`](#outcome-1)\<`D`\> +> **collect**(`settled`): [`Outcome`](#outcome-2)\<`D`\> ###### Parameters ###### settled -[`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`D`\>\> +[`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`D`\>\> ###### Returns -[`Outcome`](#outcome-1)\<`D`\> +[`Outcome`](#outcome-2)\<`D`\> *** @@ -3821,7 +4015,7 @@ Fold one settled step into the accumulated state (the loop's running deliverable ###### settled -[`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`D`\>\> +[`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`D`\>\> ###### Returns @@ -3829,7 +4023,7 @@ Fold one settled step into the accumulated state (the loop's running deliverable ##### until() -> **until**(`state`, `findings`): [`Outcome`](#outcome-1)\<`D`\> \| `null` +> **until**(`state`, `findings`): [`Outcome`](#outcome-2)\<`D`\> \| `null` The satisfiability gate: given the accumulated state + the round's trace findings, has the goal been reached? Returns the terminal deliverable when satisfied, or `null` to keep going. @@ -3847,7 +4041,7 @@ readonly `AnalystFinding`[] ###### Returns -[`Outcome`](#outcome-1)\<`D`\> \| `null` +[`Outcome`](#outcome-2)\<`D`\> \| `null` ##### label()? @@ -3949,7 +4143,7 @@ Build one judge child's task from the shared artifact under review + the judge d ##### merge() -> **merge**(`verdicts`, `artifact`): [`Outcome`](#outcome-1)\<`D`\> +> **merge**(`verdicts`, `artifact`): [`Outcome`](#outcome-2)\<`D`\> Write-only merge: fold the M settled judge verdicts into the panel's terminal `Outcome`. Pure over the drained settlements — it MUST NOT spawn, re-judge, or feed one verdict into @@ -3967,7 +4161,7 @@ readonly [`PanelVerdict`](#panelverdict)[] ###### Returns -[`Outcome`](#outcome-1)\<`D`\> +[`Outcome`](#outcome-2)\<`D`\> *** @@ -4089,7 +4283,7 @@ Build the verifier child's task from the implement child's settled candidate. ###### candidate -[`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`Candidate`\>\> +[`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`Candidate`\>\> ###### ctx @@ -4101,7 +4295,7 @@ Build the verifier child's task from the implement child's settled candidate. ##### collect() -> **collect**(`candidate`, `verdict`): [`Outcome`](#outcome-1)\<`D`\> +> **collect**(`candidate`, `verdict`): [`Outcome`](#outcome-2)\<`D`\> Project the gated (verifier-`valid`) candidate into the terminal deliverable. @@ -4109,7 +4303,7 @@ Project the gated (verifier-`valid`) candidate into the terminal deliverable. ###### candidate -[`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`Candidate`\>\> +[`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`Candidate`\>\> ###### verdict @@ -4117,7 +4311,7 @@ Project the gated (verifier-`valid`) candidate into the terminal deliverable. ###### Returns -[`Outcome`](#outcome-1)\<`D`\> +[`Outcome`](#outcome-2)\<`D`\> *** @@ -4208,7 +4402,7 @@ Build the widened child's task from the lineage the gate chose to extend. ##### synthesize() -> **synthesize**(`gathered`, `ctx`): [`Outcome`](#outcome-1)\<`D`\> +> **synthesize**(`gathered`, `ctx`): [`Outcome`](#outcome-2)\<`D`\> Synthesize the terminal deliverable from every settled lineage (selector≠judge: the single-sourced selector over the gathered children, never a re-judge). @@ -4217,7 +4411,7 @@ Synthesize the terminal deliverable from every settled lineage (selector≠judge ###### gathered -readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`D`\>\>[] +readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`D`\>\>[] ###### ctx @@ -4225,7 +4419,7 @@ readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`D`\>\>[] ###### Returns -[`Outcome`](#outcome-1)\<`D`\> +[`Outcome`](#outcome-2)\<`D`\> *** @@ -4261,7 +4455,7 @@ When true, `decide` may read `settled.verdict` directly — collides with the st ###### settled -[`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`D`\>\> +[`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`D`\>\> ###### findings @@ -4269,7 +4463,7 @@ readonly `AnalystFinding`[] ###### budget -`Readonly`\<\{ `tokensLeft`: `number`; `usdLeft`: `number`; `usdCapped`: `boolean`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +`Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> ###### Returns @@ -4300,11 +4494,11 @@ A lineage the gate may widen toward — the settled child that looked promising ###### handle -> **handle**: [`Handle`](#handle-2)\<[`Outcome`](#outcome-1)\<`D`\>\> +> **handle**: [`Handle`](#handle-2)\<[`Outcome`](#outcome-2)\<`D`\>\> ###### out -> **out**: [`Outcome`](#outcome-1) +> **out**: [`Outcome`](#outcome-2) ###### outRef @@ -4318,6 +4512,18 @@ A lineage the gate may widen toward — the settled child that looked promising > **spent**: [`Spend`](index.md#spend) +###### trace + +> **trace**: [`WorkerTraceEvidence`](index.md#workertraceevidence) + +Structured tool evidence captured before this settlement was journaled. + +###### settledAt? + +> `optional` **settledAt?**: `number` + +Epoch ms parsed from the durable settlement record when available. + ###### seq > **seq**: `number` @@ -4390,7 +4596,7 @@ Opaque root-task framing (whatever the combinator was invoked with). ##### settledSoFar -> `readonly` **settledSoFar**: readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`D`\>\>[] +> `readonly` **settledSoFar**: readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`D`\>\>[] The children this combinator has drained off `scope.next()`, in cursor order. @@ -4425,7 +4631,7 @@ explicitly NOT for steering — reading it to steer is the coupling the architec ##### settledSoFar -> `readonly` **settledSoFar**: readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`D`\>\>[] +> `readonly` **settledSoFar**: readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`D`\>\>[] ##### lastValidScore? @@ -7763,7 +7969,7 @@ The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: r ##### driver() -> **driver**(`surface`, `task`, `opts`, `budget`): [`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> +> **driver**(`surface`, `task`, `opts`, `budget`): [`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-2)\<`unknown`\>\> ###### Parameters @@ -7785,7 +7991,7 @@ The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: r ###### Returns -[`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> +[`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-2)\<`unknown`\>\> *** @@ -7942,7 +8148,7 @@ Open/close artifacts the body manages itself (e.g. one persistent handle for dep ##### scope -> `readonly` **scope**: [`Scope`](index.md#scope)\<[`Outcome`](#outcome-1)\<`unknown`\>\> +> `readonly` **scope**: [`Scope`](index.md#scope)\<[`Outcome`](#outcome-2)\<`unknown`\>\> #### Methods @@ -8880,30 +9086,6 @@ Total conserved-pool iterations = the driver + worker LLM rounds the run actuall *** -### AuthoredProfile - -What the supervisor AUTHORS per sub-task — a worker recipe (a partial `AgentProfile`). - -#### Properties - -##### name - -> **name**: `string` - -##### systemPrompt - -> **systemPrompt**: `string` - -The rich, task-specific instructions the supervisor wrote for THIS worker. - -##### model? - -> `optional` **model?**: `string` - -The model the supervisor chose for this sub-task (falls back to the run default). - -*** - ### ProfileRichnessThresholds Thresholds below which a system prompt is treated as a thin stub. Tunable per call. @@ -9022,6 +9204,31 @@ Opaque, single-use reservation handle returned by `reserve` and consumed by *** +### BudgetPoolRestore + +State recovered from a prior process before new work is admitted. `committed` is measured spend +already present in the durable journal. Each `uncertainReservation` is a child that was recorded +as started but never recorded as settled: its full declared ceiling is charged conservatively, +while the public readout remains explicitly unknown. + +#### Properties + +##### committed? + +> `readonly` `optional` **committed?**: [`Spend`](index.md#spend) + +##### uncertainReservations? + +> `readonly` `optional` **uncertainReservations?**: readonly [`Budget`](index.md#budget-4)[] + +##### absoluteDeadlineMs? + +> `readonly` `optional` **absoluteDeadlineMs?**: `number` + +Original absolute deadline from the first process. It may never slide on restart. + +*** + ### BudgetPool #### Methods @@ -9198,6 +9405,10 @@ Shared blob store — `observe_agent` reads settled outputs through it. Resolve a spawned `profile` to a worker LEAF or a driver child (the recursion seam). +##### authorizeDownMessage? + +> `readonly` `optional` **authorizeDownMessage?**: [`AuthorizeDownMessage`](#authorizedownmessage) + ##### perWorker > `readonly` **perWorker**: [`Budget`](index.md#budget-4) @@ -9248,6 +9459,13 @@ Idle time after which `observe_agent` reports a worker as stalled (a derived rea The driver's stance — a string, or built from the task (the worker-driver prompt / the generator). INJECTED so the prompt is a pluggable, optimizable role. +##### nodeTools? + +> `readonly` `optional` **nodeTools?**: readonly [`McpToolDescriptor`](mcp.md#mcptooldescriptor)[] + +Product-selected tools already bound to this exact supervisor node. The same descriptors are + served over MCP for external supervisors; this arm projects them into router ToolSpecs. + ##### extraTools? > `readonly` `optional` **extraTools?**: readonly `object`[] @@ -9347,10 +9565,11 @@ Give the driver brain a chapter-lifecycle on its OWN context window. The LLM-bra ##### onEvent? -> `readonly` `optional` **onEvent?**: (`event`) => `void` \| `Promise`\<`void`\> +> `readonly` `optional` **onEvent?**: (`event`, `record`) => `void` \| `Promise`\<`void`\> -Pass-through subscriber for every coordination bus event (settled / question / finding / - steer / answer) — what a durable caller hooks its coordination log onto. Omit = no observer. +Pass-through subscriber for every coordination bus event: settled/question/finding, + pre-delivery instruction receipts, and steer/answer delivery outcomes. A durable caller uses + this to append the coordination log. Omit = no observer. ###### Parameters @@ -9358,17 +9577,27 @@ Pass-through subscriber for every coordination bus event (settled / question / f [`CoordinationEvent`](index.md#coordinationevent) +###### record + +[`BusRecord`](#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> + ###### Returns `void` \| `Promise`\<`void`\> +##### replaySettlements? + +> `readonly` `optional` **replaySettlements?**: `boolean` + +Re-publish resume-time settlements through the awaited observer before the first brain turn. + ##### priorCoordination? > `readonly` `optional` **priorCoordination?**: [`PriorCoordination`](#priorcoordination-1) -Questions + findings a durable coordination log replayed from a prior process of this run. - Questions seed the ledger (`list_questions`, blocking-stop policy); both feed the resume - brief. Omit = fresh (every run that is not a resume). +Questions, findings, and authorized continuation receipts loaded from a prior process. + Questions seed the ledger (`list_questions`, blocking-stop policy); all three feed the resume + brief. Continuation receipts are evidence only and are never auto-delivered. Omit = fresh. ##### finalizer? @@ -9383,10 +9612,16 @@ How the settled-worker ledger becomes the run's output. Default `bestDelivered` ### PriorCoordination -What a prior process's coordination log replays into a resumed driver. +Coordination evidence loaded from prior processes of one durable supervised run. #### Properties +##### ownerId? + +> `readonly` `optional` **ownerId?**: `string` + +The owner filter used for this replay. Omitted only for the compatibility all-owner read. + ##### questions > `readonly` **questions**: readonly [`QuestionRecord`](mcp.md#questionrecord)[] @@ -9399,6 +9634,26 @@ Every question the prior process raised, with answer-status folded in, raise ord Every analyst finding the prior process published, publish order. +##### continuations + +> `readonly` **continuations**: readonly [`ContinuationInstruction`](#continuationinstruction)[] + +Every authorized continuation, in commit order. These are evidence, never replayed to a new +worker automatically. + +##### deliveryEvidence + +> `readonly` **deliveryEvidence**: readonly [`CoordinationDeliveryEvidence`](#coordinationdeliveryevidence)[] + +Delivery intent and result records in commit order, linked to receipts by `receiptId`. + +##### records + +> `readonly` **records**: readonly [`BusRecord`](#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\>[] + +Exact source-bus stamps in durable append order. Bus `seq` restarts with each process; append +order remains the cross-process replay order. + *** ### CoordinationLog @@ -9410,7 +9665,7 @@ The durable coordination side-log seam. `append` records one bus event (kinds it ##### append() -> **append**(`runId`, `event`, `at`): `Promise`\<`void`\> +> **append**(`runId`, `record`, `ownerId?`): `Promise`\<`void`\> ###### Parameters @@ -9418,11 +9673,11 @@ The durable coordination side-log seam. `append` records one bus event (kinds it `string` -###### event +###### record -[`CoordinationEvent`](index.md#coordinationevent) +[`BusRecord`](#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> -###### at +###### ownerId? `string` @@ -9432,7 +9687,7 @@ The durable coordination side-log seam. `append` records one bus event (kinds it ##### load() -> **load**(`runId`): `Promise`\<[`PriorCoordination`](#priorcoordination-1)\> +> **load**(`runId`, `ownerId?`): `Promise`\<[`PriorCoordination`](#priorcoordination-1)\> ###### Parameters @@ -9440,6 +9695,10 @@ The durable coordination side-log seam. `append` records one bus event (kinds it `string` +###### ownerId? + +`string` + ###### Returns `Promise`\<[`PriorCoordination`](#priorcoordination-1)\> @@ -9482,11 +9741,11 @@ nobody is left to read a finding, and analysts spend real compute). Returns the > **history**: () => readonly [`BusRecord`](#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\>[] -The full ordered bus-event log — observability audit + replay trail. +The full ordered bus-event log for current-process observability and audit evidence. -The full ordered log of every bus event — UP (settled / question / finding) and DOWN - (steer / answer) — the observability audit + replay trail. Each record carries seq, - timestamp, and priority. +The full ordered log of every bus event — UP (settled / question / finding), authorized + instruction receipts, and DOWN delivery outcomes (steer / answer). Each record carries seq, + timestamp, and priority. A receipt is evidence and is never auto-delivered on restart. ###### Returns @@ -9612,11 +9871,19 @@ Inject the supervisor brain directly (tests / advanced) instead of resolving it ##### supervisor? -> `readonly` `optional` **supervisor?**: `Partial`\<`Pick`\<[`SupervisorProfile`](#supervisorprofile), `"name"` \| `"systemPrompt"`\>\> +> `readonly` `optional` **supervisor?**: `object` Override the default authoring-supervisor profile (name / extra system-prompt stance). The default already carries the authoring skill; override only to add a goal or rename. +###### name? + +> `readonly` `optional` **name?**: `string` + +###### systemPrompt? + +> `readonly` `optional` **systemPrompt?**: `string` + ##### allowedModels? > `readonly` `optional` **allowedModels?**: readonly `string`[] @@ -9913,8 +10180,9 @@ Count published per event `type`. > **publish**(`event`, `opts?`): `Promise`\<[`BusRecord`](#busrecord)\<`E`\>\> -Stamp + queue the event, then deliver the stamped record to every subscriber in order. - Returns the stamped record. +Stamp the event, await every subscriber in order, then make it pull-visible. A subscriber + failure leaves the event invisible and retrying the SAME event object reuses the exact stamp. + This lets an awaited product observer commit its record before a supervisor can consume it. ###### Parameters @@ -9984,7 +10252,7 @@ readonly `E`\[`"type"`\][] > **history**(): readonly [`BusRecord`](#busrecord)\<`E`\>[] -The full ordered log of every event ever published (the audit/replay trail). +The full ordered log of every event published in this process (audit evidence, not replay). ###### Returns @@ -10568,10 +10836,11 @@ leaves it undefined: there is never a prior tree to resume, and the default stay > `readonly` `optional` **coordinationLog?**: [`CoordinationLog`](#coordinationlog) -Present only on a DURABLE context: the coordination side-log (questions + analyst findings — -the bus messages the spawn journal does not record). `supervise({ runDir })` appends to it as -they publish and replays it on resume, so a restarted coordinator keeps them. In-memory -contexts have none: nothing outlives the process to replay into. +Present only on a DURABLE context: the coordination side-log stores questions, analyst +findings, answer decisions, and authorized continuation receipts that the spawn journal does +not own. `supervise({ runDir })` appends them as they publish and loads them on resume. +Continuation receipts are evidence and are never auto-delivered to a replacement worker. +In-memory contexts have none: nothing outlives the process. *** @@ -10678,7 +10947,9 @@ Working directory for the subprocess. cli-worktree seam. A supervisor-authored `AgentProfile` driving a local coding-harness CLI (claude / codex / opencode) on its own git worktree — the leaf `createWorktreeCliExecutor` -named as data. `harness` + `repoRoot` + `taskPrompt` are required; the authored +named as data. `harness` + `repoRoot` are required; the task comes from `Executor.execute`. +`taskPrompt` remains an optional direct-call fallback for callers that execute with `undefined`. +The authored `profile.prompt.systemPrompt` + `profile.model.default` reach the harness via the §1.5 `harnessInvocation` mapper. Everything else mirrors `WorktreeCliExecutorOptions`. @@ -10694,9 +10965,9 @@ named as data. `harness` + `repoRoot` + `taskPrompt` are required; the authored Local CLI harness transport. Omit when `bridge` is set. -##### taskPrompt +##### taskPrompt? -> **taskPrompt**: `string` +> `optional` **taskPrompt?**: `string` ##### runId? @@ -10783,7 +11054,9 @@ Bridge model/harness id. Defaults to the profile's model hint when omitted. ##### agentProfile? -> `optional` **agentProfile?**: `Record`\<`string`, `unknown`\> +> `optional` **agentProfile?**: `AgentProfile` + +Canonical profile overlay merged over the spawned profile. ##### timeoutMs? @@ -10810,11 +11083,10 @@ as the harness selector (e.g. `claude-code/sonnet`, `opencode// forwarded verbatim per request — how an arm disables native tools or injects a provider search MCP. -The executor opens a RESUMABLE cli-bridge session — structurally identical to the -sandbox executor's persistent box, just local. `sessionId` is the stable -caller-owned id cli-bridge maps to the harness's internal conversation id; a -follow-up steer/resume on the SAME id continues the SAME harness session (opencode -`-s`, claude `--resume`, …). Omit it and the executor mints a stable one per spawn. +The executor opens a resumable cli-bridge session. `sessionId` identifies the +harness conversation across turns; each turn also receives its own durable run id. +A dropped HTTP reader reattaches to that exact run and explicit cancel is the only +operation allowed to stop it. Omit `sessionId` and the executor mints one per spawn. #### Properties @@ -10826,9 +11098,11 @@ follow-up steer/resume on the SAME id continues the SAME harness session (openco > **bridgeBearer**: `string` -##### model +##### model? -> **model**: `string` +> `optional` **model?**: `string` + +Fallback bridge wire id. A spawned profile may select its own harness and model. ##### cwd? @@ -10838,7 +11112,9 @@ Optional working directory forwarded to cli-bridge and persisted with the sessio ##### agentProfile? -> `optional` **agentProfile?**: `Record`\<`string`, `unknown`\> +> `optional` **agentProfile?**: `AgentProfile` + +Canonical profile overlay merged over the spawned profile. ##### timeoutMs? @@ -11233,7 +11509,7 @@ Journal/blob root key the supervisor `beginTree`'d. > `readonly` **pool**: [`BudgetPool`](#budgetpool) -The shared conserved reservation pool (one per supervised run). +The reservation pool for this scope: the root total or one nested allocated partition. ##### journal @@ -11298,6 +11574,12 @@ This scope's recursion depth (root = 0). Runtime recursion-depth ceiling — a spawn past it fails closed `depth-exceeded`. +##### maxLiveWorkers? + +> `readonly` `optional` **maxLiveWorkers?**: `number` + +Root-owned limit on live spawned workers across this scope and every nested scope. + ##### signal > `readonly` **signal**: `AbortSignal` @@ -11696,6 +11978,20 @@ Idle time that counts as stalled, passed through to the live progress read. Omit The conserved compute pool for the whole run. +##### signal? + +> `readonly` `optional` **signal?**: `AbortSignal` + +Caller-owned cancellation for the complete recursive run. Aborting it cascades through the +root scope and every live child, including acquisition and backend execution. + +##### execution? + +> `readonly` `optional` **execution?**: [`AgentExecutionRef`](#agentexecutionref) + +Trusted candidate and pursuit attribution for the root. The runtime derives profile/task +digests itself from the exact detached values it executes. + ##### backend? > `readonly` `optional` **backend?**: [`ExecutorConfig`](#executorconfig) @@ -11715,32 +12011,200 @@ The completion oracle for backend-derived workers (settled ⟺ delivered). Stron > `readonly` `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](#makeworkeragent) Override the worker seam directly (tests / advanced) instead of deriving it from `backend`. + This is caller-owned execution: profile security, spawn authorization, and recursive-driver + selection below apply only to the backend-derived worker path. `authorizeMessage` still + governs continuations sent through Runtime's coordination tools. -##### router? +##### driverBackend? -> `readonly` `optional` **router?**: [`RouterConfig`](#routerconfig) +> `readonly` `optional` **driverBackend?**: [`ExecutorConfig`](#executorconfig) -The supervisor's router substrate (`harness` null). The profile's model wins. +Run harness-brained supervisors here. Automatic execution supports a local `bridge`; a remote + sandbox requires an explicit `driveHarness` with a reachable coordination relay or tunnel. + Defaults to `backend`; separate it when managers and workers use different services. -##### brain? +##### profileSecurity? -> `readonly` `optional` **brain?**: [`ToolLoopChat`](#toolloopchat) +> `readonly` `optional` **profileSecurity?**: `AgentProfileSecurityPolicy` -Inject the supervisor brain directly (tests / advanced). +Security policy applied to every manager-authored child profile before budget reservation. + The default blocks local and remote MCP, hooks, and connection grants. Pass an explicit + allowlist to grant remote MCP hosts or other author-controlled capabilities. -##### driveHarness? +##### authorizeSpawn? -> `readonly` `optional` **driveHarness?**: [`DriveHarness`](#driveharness-1) +> `readonly` `optional` **authorizeSpawn?**: (`input`) => [`AuthorizedSpawn`](#authorizedspawn) -Run a sandboxed-harness supervisor (`harness` set). +Product authority over one complete manager-authored spawn. The callback sees the detached, + immutable profile, task, budget, label, and key together, so approving a profile cannot + authorize a different task. Return the exact allowed profile (which may be narrowed) plus + trusted candidate/pursuit attribution, or throw to refuse the whole spawn before reservation. -##### extraTools? +###### Parameters -> `readonly` `optional` **extraTools?**: readonly `object`[] +###### input -WORK tools the supervisor may call DIRECTLY — so a recursive atom can ACT (do simple work +###### profile + +`AgentProfile` + +###### parent + +`AgentProfile` + +###### parentIdentity + +[`NodeExecutionIdentity`](#nodeexecutionidentity) + +Trusted identity of the manager authorizing this exact child. + +###### parentNodeId + +`string` + +Concrete manager node; never accepted from model-authored tool arguments. + +###### assignmentId + +`string` + +Stable manager-scoped assignment, including deterministic unkeyed siblings. + +###### task + +`unknown` + +###### budget + +[`Budget`](index.md#budget-4) + +###### label + +`string` + +###### key? + +`string` + +###### depth + +`number` + +###### Returns + +[`AuthorizedSpawn`](#authorizedspawn) + +##### authorizeMessage? + +> `readonly` `optional` **authorizeMessage?**: (`input`) => [`AuthorizedDownMessage`](#authorizeddownmessage) + +Product authority over every continuation sent to a live child. When spawn authorization is +enabled, omitting this refuses steer/answer instructions instead of silently extending the +authorized task. The exact worker identity and detached bytes are recorded before delivery. + +###### Parameters + +###### input + +[`DownMessageAuthorizationInput`](#downmessageauthorizationinput) & `object` + +###### Returns + +[`AuthorizedDownMessage`](#authorizeddownmessage) + +##### isDriverProfile? + +> `readonly` `optional` **isDriverProfile?**: (`input`) => `boolean` + +Decide whether an authorized child becomes another supervisor. By default only + `metadata.role === 'driver'` does; products may bind this to their own authority record. + +###### Parameters + +###### input + +###### profile + +`AgentProfile` + +###### parent + +`AgentProfile` + +###### depth + +`number` + +###### Returns + +`boolean` + +##### router? + +> `readonly` `optional` **router?**: [`RouterConfig`](#routerconfig) + +The supervisor's router substrate (`profile.harness` omitted or `cli-base`). The profile's + model wins. + +##### brain? + +> `readonly` `optional` **brain?**: [`ToolLoopChat`](#toolloopchat) + +Inject the supervisor brain directly (tests / advanced). + +##### driveHarness? + +> `readonly` `optional` **driveHarness?**: [`DriveHarness`](#driveharness-1) + +Run an external-harness supervisor explicitly. Required for a remote sandbox; optional as a + caller-owned override for a local bridge. + +##### driveHarnessMaterialization? + +> `readonly` `optional` **driveHarnessMaterialization?**: [`ProfileMaterializationContract`](agent.md#profilematerializationcontract) + +Required with a custom `driveHarness`: declares which complete AgentProfile axes that path +really applies. Built-in bridge driving supplies its own full-profile contract. + +##### resolveSupervisorTools? + +> `readonly` `optional` **resolveSupervisorTools?**: [`ResolveSupervisorTools`](#resolvesupervisortools-1) + +Resolve product-owned tools from the exact trusted manager context. The same descriptors and +handlers are bound to router and external-harness managers; resolution happens once per node. + +##### onCoordinationEvent? + +> `readonly` `optional` **onCoordinationEvent?**: (`context`, `eventId`, `record`) => `void` \| `Promise`\<`void`\> + +Awaited product transaction hook for every coordination record. `eventId` is stable across a +lost acknowledgement and durable restart; the record is not pull-visible until this commits. + +###### Parameters + +###### context + +[`SupervisorNodeContext`](#supervisornodecontext) + +###### eventId + +`` `sha256:${string}` `` + +###### record + +[`BusRecord`](#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> + +###### Returns + +`void` \| `Promise`\<`void`\> + +##### extraTools? + +> `readonly` `optional` **extraTools?**: readonly `object`[] + +WORK tools the supervisor may call DIRECTLY — so a recursive atom can ACT (do simple work itself) OR SPAWN (delegate when it needs parallelism), not be a pure manager. Pair with - `executeExtraTool`. Router arm only (`harness` null). + `executeExtraTool`. Router arm only (`profile.harness` omitted or `cli-base`). ##### executeExtraTool? @@ -11772,9 +12236,9 @@ Per-child budget reserved on each spawn. Defaults to a quarter of the pool's tok > `readonly` `optional` **maxLiveWorkers?**: `number` -Hard cap on simultaneously-LIVE workers — `spawn_agent` fails closed once this many are in - flight. The conserved pool bounds TOTAL work; this bounds SIMULTANEOUS work (live boxes/ - sandboxes a real fleet runs at once). Omit/`<= 0` = no cap (the pool stays the only fence). +Hard cap on simultaneously executing spawned workers across the WHOLE recursive tree. The + root is excluded; nested drivers and leaves share one allocation, so recursion cannot multiply + the cap. Omit/`<= 0` = no cap (the conserved pool stays the only bound). ##### analysts? @@ -11823,17 +12287,23 @@ Worker output store. Defaults to in-memory. Make the run DURABLE: journal + result blobs + the coordination side-log are file-backed under this directory (`createFileRunContext`), fsynced per write, and the supervisor reads the prior -tree first. Re-running with the same `runDir` AND the same `runId` resumes, and the built-in -driver is resume-AWARE out of the box: the children that already settled are replayed onto +tree first. Re-running with the same `runDir` AND the same `runId` resumes only when the exact +root profile/task identity and declared budget match. The original absolute deadline and prior +measured spend are restored before new admission. The built-in driver is resume-aware: children +that already settled, including their exact execution identities, are replayed onto `Scope.resume` (and into the driver's settled ledger + its first context), keyed assignments (`spawn_agent`'s `key`) resolve to their committed results instead of re-running, pending -waits re-arm on their original deadlines, prior questions/findings replay from the -coordination log, and the finalize spans both processes' work. Unset = in-memory, fresh -every call. +waits re-arm on their original deadlines, and the coordination log loads prior questions, +findings, and instruction receipts. The router arm receives all three in its resume brief; the +external arm seeds prior questions while findings and receipts remain in the durable log. +Instruction receipts are evidence and are never delivered automatically to a replacement +worker. The final result spans both processes' work. Unset = in-memory, fresh every call. The boundary that remains: work that was IN FLIGHT when the process died is not recovered — -the built-in executors cannot re-attach to a dead process's executions, so those assignments -resume as explicitly lost/in-doubt and re-run (reported, never silent). +the built-in executors cannot re-attach to a dead process's executions. Each such assignment +resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively, +and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining +capacity, so restart cannot mint a fresh budget or slide the original absolute deadline. `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single resumable run per directory but collides across concurrent runs sharing one `runDir`. @@ -11933,34 +12403,128 @@ How the settled-worker ledger becomes the run's output. Default `bestDelivered` *** -### SupervisorProfile +### AuthorizedSpawn -The supervisor's profile — the subset of an `AgentProfile` that selects + shapes its brain. - `harness` is the backend-as-data discriminant; `systemPrompt` is the standing instruction. +The product-authorized result for one complete spawn request. Attribution is never accepted +from the manager itself; it enters only through this trusted callback. #### Properties -##### name? +##### profile -> `readonly` `optional` **name?**: `string` +> `readonly` **profile**: `AgentProfile` -##### harness? +##### execution? -> `readonly` `optional` **harness?**: `string` \| `null` +> `readonly` `optional` **execution?**: [`AgentExecutionRef`](#agentexecutionref) -null/undefined → router brain (in-process tool-loop); a coding-CLI harness → sandboxed brain. +*** -##### model? +### SupervisorNodeContext -> `readonly` `optional` **model?**: `string` +Trusted run/node identity Runtime binds to one manager. Model-authored tool arguments cannot + provide or replace any of these fields. -The router model when the brain is router-driven (falls back to the deps router config). +#### Properties -##### systemPrompt? +##### runId -> `readonly` `optional` **systemPrompt?**: `string` +> `readonly` **runId**: `string` + +##### runNamespace + +> `readonly` **runNamespace**: `string` + +Stable across a durable restart; unique per in-memory invocation. + +##### nodeId + +> `readonly` **nodeId**: `string` + +Concrete Scope node that owns this manager's coordination stream. + +##### ownerId + +> `readonly` **ownerId**: `string` + +Stable identity of this manager's coordination stream. + +##### depth + +> `readonly` **depth**: `number` + +##### identity + +> `readonly` **identity**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +##### assignmentId? + +> `readonly` `optional` **assignmentId?**: `string` -The standing instructions ("you delegate, you do not solve"). +Assignment identity within the parent manager; absent only for the root. + +##### profile + +> `readonly` **profile**: `AgentProfile` + +##### task + +> `readonly` **task**: `unknown` + +*** + +### SupervisorToolDescriptor + +One product-owned tool. It reuses the canonical MCP descriptor fields while Runtime supplies + the trusted node context as a separate argument and binds the result for either transport. + +#### Extends + +- `Omit`\<[`McpToolDescriptor`](mcp.md#mcptooldescriptor), `"handler"`\> + +#### Properties + +##### name + +> **name**: `string` + +###### Inherited from + +[`McpToolDescriptor`](mcp.md#mcptooldescriptor).[`name`](mcp.md#name-2) + +##### description + +> **description**: `string` + +###### Inherited from + +[`McpToolDescriptor`](mcp.md#mcptooldescriptor).[`description`](mcp.md#description) + +##### inputSchema + +> **inputSchema**: `Record`\<`string`, `unknown`\> + +###### Inherited from + +[`McpToolDescriptor`](mcp.md#mcptooldescriptor).[`inputSchema`](mcp.md#inputschema) + +##### handler + +> `readonly` **handler**: (`raw`, `context`) => `Promise`\<`unknown`\> + +###### Parameters + +###### raw + +`unknown` + +###### context + +[`SupervisorNodeContext`](#supervisornodecontext) + +###### Returns + +`Promise`\<`unknown`\> *** @@ -11978,6 +12542,12 @@ The standing instructions ("you delegate, you do not solve"). Resolve a spawned worker `profile` to a leaf agent — the recursion seam (same for both arms). +##### authorizeDownMessage? + +> `readonly` `optional` **authorizeDownMessage?**: [`AuthorizeDownMessage`](#authorizedownmessage) + +Product authorization for every down-leg continuation to a child. + ##### perWorker > `readonly` **perWorker**: [`Budget`](index.md#budget-4) @@ -11996,7 +12566,8 @@ Hard cap on simultaneously-LIVE workers across both arms — `spawn_agent` fails > `readonly` `optional` **router?**: [`RouterConfig`](#routerconfig) -Router substrate for a router-brained supervisor (`harness` null). The profile's model wins. +Router substrate for a router-brained supervisor (`harness` omitted or `cli-base`). The + profile's model wins. ##### brain? @@ -12008,7 +12579,32 @@ Inject the brain directly (tests / advanced) instead of resolving `routerBrain` > `readonly` `optional` **driveHarness?**: [`DriveHarness`](#driveharness-1) -Required for a sandboxed-harness supervisor (`harness` set): runs the harness as the driver. +Required to run an external-harness supervisor: runs the harness as the driver. + +##### nodeContext? + +> `readonly` `optional` **nodeContext?**: [`SupervisorNodeContextSeed`](#supervisornodecontextseed) + +Trusted identity for this manager. Required with node-scoped tools or observation. + +##### resolveSupervisorTools? + +> `readonly` `optional` **resolveSupervisorTools?**: [`ResolveSupervisorTools`](#resolvesupervisortools-1) + +Resolve product-owned tools for this exact manager. Static `extraTools` remain a router-only + compatibility seam and deliberately receive no new recursive authority. + +##### observeNodeEvent? + +> `readonly` `optional` **observeNodeEvent?**: [`ObserveSupervisorNodeEvent`](#observesupervisornodeevent) + +Awaited product observation, enriched with this manager's actual live node context. + +##### replaySettlements? + +> `readonly` `optional` **replaySettlements?**: `boolean` + +Replay resume-time settlements through `observeNodeEvent` before the manager starts. ##### extraTools? @@ -12102,7 +12698,7 @@ Give the supervisor brain a chapter-lifecycle on its OWN context window (router ##### onEvent? -> `readonly` `optional` **onEvent?**: (`event`) => `void` \| `Promise`\<`void`\> +> `readonly` `optional` **onEvent?**: (`event`, `record`) => `void` \| `Promise`\<`void`\> Pass-through subscriber for every coordination bus event (both arms) — the seam a durable caller hooks its coordination log onto. @@ -12113,6 +12709,10 @@ Pass-through subscriber for every coordination bus event (both arms) — the sea [`CoordinationEvent`](index.md#coordinationevent) +###### record + +[`BusRecord`](#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> + ###### Returns `void` \| `Promise`\<`void`\> @@ -12121,8 +12721,21 @@ Pass-through subscriber for every coordination bus event (both arms) — the sea > `readonly` `optional` **priorCoordination?**: [`PriorCoordination`](#priorcoordination-1) -Questions + findings replayed from a prior process of this run (a durable coordination log). - Router arm: seeds the question ledger + the resume brief. Sandbox arm: seeds the ledger. +Questions, findings, and authorized continuation receipts loaded from a prior process. + Router arm: questions seed the ledger and all evidence enters the resume brief. External arm: + questions seed the ledger; receipts remain durable evidence and are never auto-delivered. + +##### loadPriorCoordination? + +> `readonly` `optional` **loadPriorCoordination?**: () => `Promise`\<[`PriorCoordination`](#priorcoordination-1)\> + +Deferred owner-scoped replay for a recursive supervisor. Its stable owner is known while the +parent authorizes the child, but loading remains asynchronous; Runtime calls this before the +nested brain can publish or act on coordination state. + +###### Returns + +`Promise`\<[`PriorCoordination`](#priorcoordination-1)\> ##### finalizer? @@ -12133,6 +12746,22 @@ How the settled ledger becomes the run's output (both arms). Default `bestDelive *** +### WorkerToolTraceArtifact + +Bytes stored under `WorkerTraceEvidence.traceRef`. + +#### Properties + +##### schemaVersion + +> `readonly` **schemaVersion**: `1` + +##### spans + +> `readonly` **spans**: readonly `ToolSpan`[] + +*** + ### ToolStepInput #### Properties @@ -12243,122 +12872,315 @@ The minimal box surface this needs: list a session's messages (incl. mid-turn pa *** -### TrajectoryAnalysis +### TrajectoryAnalysis + +#### Properties + +##### trajectory + +> `readonly` **trajectory**: `Trajectory` + +Structured run summary (tool-call count, step order). Steps carry a single timestamp, so per-span + duration is 0; loop/waste detection keys on call PATTERNS + cross-span windows, not durations. + +##### stuckLoop + +> `readonly` **stuckLoop**: `StuckLoopReport` + +Full-run repeated-call view (total occurrences + window) — allows one intervening call so it +catches a loop the online consecutive detector interleaves past. + +##### toolWaste + +> `readonly` **toolWaste**: `ToolWasteReport` + +Wasted-vs-total tool-call ratio for the run. + +*** + +### WaitOpts + +Options for `Scope.wait`. `label` is the wait's identity within its parent scope — it is what + a resumed run matches to re-adopt a journaled, still-unfired wait, so it must be stable across + processes (a label derived from wall-clock would resume as a NEW wait). + +#### Properties + +##### label + +> `readonly` **label**: `string` + +*** + +### Agent + +One self-similar atom. A leaf is an `Agent` that never calls `scope.spawn`; a driver +is an `Agent` whose `act` spawns children and reacts to them via `scope.next()`. An +analyst is an `Agent` whose task is "read these traces → findings" — `where` it runs +is its executor, not a separate type. + +`act` MUST be replay-safe: it may read `verdict`, `spent`, and `out` (rehydrated by +`outRef`) off each `Settled`; it MUST NOT read `Date.now`, `Math.random`, or any +unordered collection. `scope.next()` delivers strictly in recorded `seq` order. + +#### Type Parameters + +##### Task + +`Task` + +##### Out + +`Out` + +#### Properties + +##### name + +> `readonly` **name**: `string` + +#### Methods + +##### act() + +> **act**(`task`, `scope`): `Promise`\<`Out`\> + +###### Parameters + +###### task + +`Task` + +###### scope + +[`Scope`](index.md#scope)\<`Out`\> + +###### Returns + +`Promise`\<`Out`\> + +*** + +### ExecutorAccounting + +Split used by a recursive executor when journaled child work differs from the full amount +reconciled against its parent reservation. + +#### Properties + +##### reported + +> `readonly` **reported**: [`Spend`](index.md#spend) + +##### reservation + +> `readonly` **reservation**: [`Spend`](index.md#spend) + +*** + +### ExecutorResult + +Terminal artifact of a one-shot `Executor.execute`. + +#### Type Parameters + +##### Out + +`Out` + +#### Properties + +##### outRef + +> **outRef**: `string` + +##### out + +> **out**: `Out` + +##### verdict? + +> `optional` **verdict?**: `DefaultVerdict` + +##### spent + +> **spent**: [`Spend`](index.md#spend) + +*** + +### AgentExecutionRef + +Caller-owned identity beyond the exact profile/task bytes Scope can compute itself. + +#### Extended by + +- [`NodeExecutionIdentity`](#nodeexecutionidentity) + +#### Properties + +##### candidateDigest? + +> `readonly` `optional` **candidateDigest?**: `` `sha256:${string}` `` + +##### correlation? + +> `readonly` `optional` **correlation?**: `Readonly`\<`Record`\<`string`, `string`\>\> + +*** + +### NodeExecutionIdentity + +Durable identity of one realized node. Missing digests mean the input was not canonical JSON. + +#### Extends + +- [`AgentExecutionRef`](#agentexecutionref) + +#### Properties + +##### candidateDigest? + +> `readonly` `optional` **candidateDigest?**: `` `sha256:${string}` `` + +###### Inherited from + +[`AgentExecutionRef`](#agentexecutionref).[`candidateDigest`](#candidatedigest) + +##### correlation? + +> `readonly` `optional` **correlation?**: `Readonly`\<`Record`\<`string`, `string`\>\> + +###### Inherited from + +[`AgentExecutionRef`](#agentexecutionref).[`correlation`](#correlation) + +##### profileDigest? + +> `readonly` `optional` **profileDigest?**: `` `sha256:${string}` `` + +##### taskDigest? + +> `readonly` `optional` **taskDigest?**: `` `sha256:${string}` `` + +*** + +### MaterializedExecutionIdentity + +External execution identity that operators can use to join this node to its backend. #### Properties -##### trajectory +##### kind -> `readonly` **trajectory**: `Trajectory` +> `readonly` **kind**: `string` -Structured run summary (tool-call count, step order). Steps carry a single timestamp, so per-span - duration is 0; loop/waste detection keys on call PATTERNS + cross-span windows, not durations. +Backend-native identity kind, for example `request`, `session`, `run`, `process`, or `tree`. -##### stuckLoop +##### id -> `readonly` **stuckLoop**: `StuckLoopReport` +> `readonly` **id**: `string` -Full-run repeated-call view (total occurrences + window) — allows one intervening call so it -catches a loop the online consecutive detector interleaves past. +*** -##### toolWaste +### ExecutorMaterialization -> `readonly` **toolWaste**: `ToolWasteReport` +Data-only declaration from trusted executor code about the exact sealed plan `execute` uses. +Scope snapshots this value and computes the durable receipt; callers never provide digests. -Wasted-vs-total tool-call ratio for the run. +#### Properties -*** +##### effectiveProfile -### WaitOpts +> `readonly` **effectiveProfile**: `AgentProfile` -Options for `Scope.wait`. `label` is the wait's identity within its parent scope — it is what - a resumed run matches to re-adopt a journaled, still-unfired wait, so it must be stable across - processes (a label derived from wall-clock would resume as a NEW wait). +Complete profile after trusted runtime-owned attachments or backend overlays were applied. -#### Properties +##### backend -##### label +> `readonly` **backend**: `string` -> `readonly` **label**: `string` +Concrete backend or harness selected for this run. -*** +##### model -### Agent +> `readonly` **model**: [`MaterializedModelIdentity`](#materializedmodelidentity) -One self-similar atom. A leaf is an `Agent` that never calls `scope.spawn`; a driver -is an `Agent` whose `act` spawns children and reacts to them via `scope.next()`. An -analyst is an `Agent` whose task is "read these traces → findings" — `where` it runs -is its executor, not a separate type. +Exact selected model, or an explicit unknown reason. -`act` MUST be replay-safe: it may read `verdict`, `spent`, and `out` (rehydrated by -`outRef`) off each `Settled`; it MUST NOT read `Date.now`, `Math.random`, or any -unordered collection. `scope.next()` delivers strictly in recorded `seq` order. +##### execution -#### Type Parameters +> `readonly` **execution**: [`MaterializedExecutionIdentity`](#materializedexecutionidentity) -##### Task +Backend-native session/run/request/process identity. -`Task` +##### materializer -##### Out +> `readonly` **materializer**: `string` -`Out` +Named implementation that turns the effective profile into executable backend inputs. -#### Properties +##### plan -##### name +> `readonly` **plan**: `unknown` -> `readonly` **name**: `string` +Finite JSON describing the exact materialization plan. Persisted by digest only. -#### Methods +##### platformAttachments? -##### act() +> `readonly` `optional` **platformAttachments?**: `unknown` -> **act**(`task`, `scope`): `Promise`\<`Out`\> +Trusted runtime-only attachments, such as the coordination MCP. Persisted by digest only. -###### Parameters +*** -###### task +### ExecutorExecutionBinding -`Task` +Volatile execution routing that is true for one attempt but is not profile identity. The full +binding is hashed and discarded; only the safe structural descriptor is journaled. -###### scope +#### Properties -[`Scope`](index.md#scope)\<`Out`\> +##### attemptId -###### Returns +> `readonly` **attemptId**: `string` -`Promise`\<`Out`\> +##### binding -*** +> `readonly` **binding**: `unknown` -### ExecutorResult +##### descriptor -Terminal artifact of a one-shot `Executor.execute`. +> `readonly` **descriptor**: `Readonly`\<`Record`\<`string`, `string` \| `number` \| `boolean` \| `null`\>\> -#### Type Parameters +*** -##### Out +### ExecutorNodeContext -`Out` +Kernel-owned context for the concrete supervised node a factory is constructing. #### Properties -##### outRef +##### rootId -> **outRef**: `string` +> `readonly` **rootId**: `string` -##### out +##### parentId -> **out**: `Out` +> `readonly` **parentId**: `string` -##### verdict? +##### nodeId -> `optional` **verdict?**: `DefaultVerdict` +> `readonly` **nodeId**: `string` -##### spent +##### attemptId -> **spent**: [`Spend`](index.md#spend) +> `readonly` **attemptId**: `string` + +Kernel-minted identity for this concrete execution attempt. + +##### identity? + +> `readonly` `optional` **identity?**: [`NodeExecutionIdentity`](#nodeexecutionidentity) *** @@ -12374,6 +13196,12 @@ Construction context handed to a `ExecutorFactory` — the seams a built-in need > `readonly` **signal**: `AbortSignal` +##### node? + +> `readonly` `optional` **node?**: [`ExecutorNodeContext`](#executornodecontext) + +Present when Scope constructs the executor for a supervised node. + ##### seams > `readonly` **seams**: `Readonly`\<`Record`\<`string`, `unknown`\>\> @@ -12394,6 +13222,13 @@ Opaque seams the registry threads through; a built-in narrows what it needs. > `readonly` **label**: `string` +##### assignmentId? + +> `readonly` `optional` **assignmentId?**: `string` + +Manager-scoped semantic assignment identity. Unlike `key`, this names every spawn, including +unkeyed siblings, so product traces can join authorization, node, and backend execution. + ##### restart? > `readonly` `optional` **restart?**: [`Restart`](#restart) @@ -12445,6 +13280,30 @@ mid-acquire never leaks (M1). > `readonly` **status**: [`NodeStatus`](#nodestatus) +##### assignmentId? + +> `readonly` `optional` **assignmentId?**: `string` + +Manager-scoped assignment identity supplied at admission. + +##### identity? + +> `readonly` `optional` **identity?**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +Durable identity of the authorized profile/task/candidate represented by this handle. + +##### materialization? + +> `readonly` `optional` **materialization?**: [`ProfileMaterializationReceipt`](#profilematerializationreceipt) + +Stable execution plan once Runtime has committed it. + +##### executionBindings? + +> `readonly` `optional` **executionBindings?**: readonly [`ExecutionBindingReceipt`](#executionbindingreceipt)[] + +Immutable per-attempt backend bindings committed so far, oldest first. + ##### \_\_out? > `readonly` `optional` **\_\_out?**: `Out` @@ -12551,6 +13410,12 @@ What the journal proves about one keyed assignment at resume time. > `readonly` **label**: `string` +##### identity? + +> `readonly` `optional` **identity?**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +Identity recorded when this key was first admitted. Every reuse must match it exactly. + ##### state > `readonly` **state**: `"completed"` \| `"down"` \| `"in-doubt"` @@ -12591,6 +13456,34 @@ The rehydrated settlement; absent exactly when `state` is `'in-doubt'`. > `readonly` **budget**: [`Budget`](index.md#budget-4) +##### assignmentId? + +> `readonly` `optional` **assignmentId?**: `string` + +Manager-scoped assignment identity, including deterministic ids for unkeyed siblings. + +##### identity? + +> `readonly` `optional` **identity?**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +##### materialization? + +> `readonly` `optional` **materialization?**: [`ProfileMaterializationReceipt`](#profilematerializationreceipt) + +Kernel-owned execution evidence. `unknown` is distinct from a known zero/empty plan. + +##### executionBindings? + +> `readonly` `optional` **executionBindings?**: readonly [`ExecutionBindingReceipt`](#executionbindingreceipt)[] + +Immutable attempt bindings, oldest first. A retried/resumed node may have more than one. + +##### settledAt? + +> `readonly` `optional` **settledAt?**: `number` + +Epoch ms of the terminal journal record; absent while live or when legacy evidence lacks it. + ##### spent > `readonly` **spent**: [`Spend`](index.md#spend) @@ -12603,6 +13496,12 @@ Conserved spend so far for this node. `outRef` once the node is `done` (the replay/result pointer). +##### trace? + +> `readonly` `optional` **trace?**: [`WorkerTraceEvidence`](index.md#workertraceevidence) + +Present on terminal executor nodes; legacy records carry an explicit unavailable reason. + *** ### TreeView @@ -12747,6 +13646,19 @@ Content-addressed result blobs (the `outRef` → artifact map) backing the repla The root conserved-pool ceiling (tokens + usd + iterations + deadline). +##### rootIdentity? + +> `readonly` `optional` **rootIdentity?**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +Exact root profile/task identity supplied by the one-call composition surface. + +##### rootMaterialization? + +> `readonly` `optional` **rootMaterialization?**: [`RootMaterialization`](#rootmaterialization) + +Trusted composition evidence for a root whose `act` drives an external backend. A generic + root omits it and is durably marked unknown; model-facing Scope never receives this writer. + ##### runId > `readonly` **runId**: `string` @@ -12785,6 +13697,13 @@ Predicate resolution for `poll` wait-states (`Scope.wait`). A `poll` names its p Runtime recursion-depth ceiling (paired with the conserved pool per R3). +##### maxLiveWorkers? + +> `readonly` `optional` **maxLiveWorkers?**: `number` + +Hard tree-wide cap on simultaneously executing spawned workers. The root is excluded; every + nested driver and leaf shares this one allocation. Omit/`<= 0` leaves worker count uncapped. + ##### maxRestarts? > `readonly` `optional` **maxRestarts?**: `number` @@ -12930,7 +13849,7 @@ Default impl returns false for every settlement (flat — never widens). ###### budget -`Readonly`\<\{ `tokensLeft`: `number`; `usdLeft`: `number`; `usdCapped`: `boolean`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +`Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> ###### Returns @@ -13079,10 +13998,9 @@ Absolute path to the git checkout the worktree is cut from. **`Experimental`** The supervisor-authored prompt/model plus materializable structural resources. -`model.default` selects the one-shot model; `small`, `provider`, and `metadata` remain hints. -Resource failures are fatal regardless of `resources.failOnError`. -Tools, permissions, connections, confidential execution, modes, and extensions fail closed. -Harness-specific nested controls that the pinned materializer cannot preserve also fail closed. +`model.default` selects the one-shot model. Routing-only model hints, placement concerns, +provider extensions, and `resources.failOnError` fail before execution because this path +cannot honor them. Harness-specific values the materializer cannot preserve also fail closed. ##### harness @@ -13092,13 +14010,14 @@ Harness-specific nested controls that the pinned materializer cannot preserve al Local CLI for this leaf. This explicit choice overrides `profile.harness`. -##### taskPrompt +##### taskPrompt? -> **taskPrompt**: `string` +> `optional` **taskPrompt?**: `string` **`Experimental`** -The per-task instruction handed to the harness (composed under the system prompt). +Default instruction for direct `execute(undefined, signal)` calls. An execution-time task + is authoritative. Omit when the caller always supplies the task to `execute`. ##### runId? @@ -13278,6 +14197,31 @@ The supervisor-authored `AgentProfile` (systemPrompt + model reach the harness v Which local harness CLI drives this leaf. +##### budgetExempt? + +> `optional` **budgetExempt?**: `boolean` + +**`Experimental`** + +Require measured usage from this leaf. Budgeted supervision refuses the default unmetered + local-CLI mode; set false only when the selected runner actually returns token usage. + +##### codexReproducible? + +> `optional` **codexReproducible?**: `boolean` + +**`Experimental`** + +Run Codex through its measured, isolated JSONL path. This implies `budgetExempt: false`. + +##### codexReadDeniedPaths? + +> `optional` **codexReadDeniedPaths?**: readonly `string`[] + +**`Experimental`** + +Host paths denied to a reproducible Codex leaf. + ##### runId? > `optional` **runId?**: `string` @@ -15119,15 +16063,45 @@ Present when a commit was attempted (valid, or `commitOnInvalid`). ## Type Aliases +### DownMessageDeliveryOutcome + +> **DownMessageDeliveryOutcome** = `"delivered"` \| `"unknown-worker"` \| `"already-settled"` \| `"runtime-has-no-inbox"` \| `"scope-stopped"` \| `"runtime-error"` + +The exact result of one parent→child delivery attempt. + +*** + +### AuthorizeDownMessage + +> **AuthorizeDownMessage** = (`input`) => [`AuthorizedDownMessage`](#authorizeddownmessage) + +Product decision over an exact continuation before it is durably recorded or delivered. + +#### Parameters + +##### input + +[`DownMessageAuthorizationInput`](#downmessageauthorizationinput) + +#### Returns + +[`AuthorizedDownMessage`](#authorizeddownmessage) + +*** + ### MakeWorkerAgent -> **MakeWorkerAgent** = (`profile`) => [`Agent`](#agent-1)\<`unknown`, `unknown`\> +> **MakeWorkerAgent** = (`profile`, `context?`) => [`Agent`](#agent-1)\<`unknown`, `unknown`\> #### Parameters ##### profile -`unknown` +`AgentProfile` + +##### context? + +[`WorkerSpawnContext`](#workerspawncontext) #### Returns @@ -15304,7 +16278,7 @@ Builds a frozen `Persona`, failing loud on the executors-supplied invariant (nei ### LoopShape -> **LoopShape**\<`Task`, `D`\> = (`ctx`) => [`Agent`](#agent-1)\<`Task`, [`Outcome`](#outcome-1)\<`D`\>\> +> **LoopShape**\<`Task`, `D`\> = (`ctx`) => [`Agent`](#agent-1)\<`Task`, [`Outcome`](#outcome-2)\<`D`\>\> A reusable act-body factory. Given the persona's content + seams (`ShapeContext`), it returns the root `Agent>` whose `act` decomposes the task, fans out @@ -15330,13 +16304,13 @@ synthesizes the terminal `Outcome`. The shape is STRUCTURE; the persona is CO #### Returns -[`Agent`](#agent-1)\<`Task`, [`Outcome`](#outcome-1)\<`D`\>\> +[`Agent`](#agent-1)\<`Task`, [`Outcome`](#outcome-2)\<`D`\>\> *** ### RunPersonified -> **RunPersonified** = \<`Task`, `D`\>(`options`) => `Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<[`Outcome`](#outcome-1)\<`D`\>\>\> +> **RunPersonified** = \<`Task`, `D`\>(`options`) => `Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<[`Outcome`](#outcome-2)\<`D`\>\>\> The composed run signature. @@ -15358,7 +16332,7 @@ The composed run signature. #### Returns -`Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<[`Outcome`](#outcome-1)\<`D`\>\>\> +`Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<[`Outcome`](#outcome-2)\<`D`\>\>\> *** @@ -15416,7 +16390,7 @@ the persona carries the domain. ### FanoutWinnerSelector -> **FanoutWinnerSelector**\<`D`\> = (`iterations`) => \{ `output?`: [`Outcome`](#outcome-1)\<`D`\>; \} \| `undefined` +> **FanoutWinnerSelector**\<`D`\> = (`iterations`) => \{ `output?`: [`Outcome`](#outcome-2)\<`D`\>; \} \| `undefined` A winner-selection strategy: argmax/sort over the gathered child iterations (each output is the child's `Outcome`), returning the chosen iteration or `undefined` when none qualifies. @@ -15431,11 +16405,11 @@ A winner-selection strategy: argmax/sort over the gathered child iterations (eac ##### iterations -[`Iteration`](#iteration-1)\<`unknown`, [`Outcome`](#outcome-1)\<`D`\>\>[] +[`Iteration`](#iteration-1)\<`unknown`, [`Outcome`](#outcome-2)\<`D`\>\>[] #### Returns -\{ `output?`: [`Outcome`](#outcome-1)\<`D`\>; \} \| `undefined` +\{ `output?`: [`Outcome`](#outcome-2)\<`D`\>; \} \| `undefined` *** @@ -15710,7 +16684,7 @@ judge/verdict/score scheme is rejected. Fail loud — a tainted finding aborts. ##### root -[`NodeId`](#nodeid-1) +[`NodeId`](#nodeid-3) ##### options? @@ -15969,72 +16943,181 @@ Provider-neutral conversation records read by structural candidate extraction. *** +### AuthoredProfile + +> **AuthoredProfile** = `AgentProfile` & `object` + +What the supervisor AUTHORS per sub-task: one complete canonical profile whose name and + task-specific system prompt are present. Every other `AgentProfile` axis is preserved exactly. + +#### Type Declaration + +##### name + +> `readonly` **name**: `string` + +##### prompt + +> `readonly` **prompt**: `AgentProfilePrompt` & `object` + +###### Type Declaration + +###### systemPrompt + +> `readonly` **systemPrompt**: `string` + +*** + ### BudgetReadout -> **BudgetReadout** = `Readonly`\<\{ `tokensLeft`: `number`; `usdLeft`: `number`; `usdCapped`: `boolean`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +> **BudgetReadout** = `Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> Post-reservation pool readout — the shape `Scope.budget` exposes. `tokensLeft`, `usdLeft`, and `reservedTokens` reflect committed-but-unsettled reservations; `deadlineMs` is the ABSOLUTE wall-clock deadline (0 when the root set none). + `iterationsLeft` is the remaining iteration capacity. `tokensKnown`/`usdKnown` are false when + any prior or current execution lacks measured telemetry; the numeric remainder may still be a + safe admission bound when an in-doubt child has already been charged at its full reservation. `usdCapped` distinguishes a real `usdLeft <= 0` exhaustion from an uncapped pool (which always reads `usdLeft: 0`) — the in-loop guard needs it to bound a usd-capped driver. *** -### DispatchStopReason +### CoordinationOwnerId + +> **CoordinationOwnerId** = `string` + +Stable identity of the supervisor that owns one coordination stream. High-level supervision +derives it from the exact root/child execution identity plus its parent assignment. + +*** + +### CoordinationDeliveryEvidence + +> **CoordinationDeliveryEvidence** = `Extract`\<[`CoordinationEvent`](index.md#coordinationevent), \{ `type`: `"delivery-attempt"` \| `"steer"` \| `"answer"`; \}\> + +Durable delivery evidence retained in commit order. An attempt without a later event carrying +the same `receiptId` has an unknown outcome after a crash and is never replayed. + +*** + +### DispatchStopReason + +> **DispatchStopReason** = `"drained"` \| `"not-admitted"` \| `"stopped"` \| `"aborted"` + +Why the dispatcher stopped admitting work. `drained` = the queue ran dry (the ordinary end); + `not-admitted` = the conserved pool or the depth ceiling refused a spawn; `stopped` = the + caller's `shouldStop` returned true; `aborted` = the scope's signal fired. + +*** + +### RunContext + +> **RunContext** = [`InMemoryRunContext`](#inmemoryruncontext) + +The stores a supervised run needs, in-memory or file-backed. `InMemoryRunContext` is the + historical name for the same shape. + +*** + +### ExecutorConfig + +> **ExecutorConfig** = `object` & [`RouterSeam`](#routerseam) \| `object` & [`RouterToolsSeam`](#routertoolsseam) \| `object` & [`BridgeSeam`](#bridgeseam) \| `object` & [`CliSeam`](#cliseam) \| `object` & [`CliWorktreeSeam`](#cliworktreeseam) \| `object` & [`ProviderSeam`](#providerseam) \| `object` & [`PiSeam`](#piseam) \| `object` & [`SandboxSeam`](#sandboxseam) + +Config for [createExecutor](#createexecutor): the backend is DATA — the cost dial a profile, +an experiment config, or a replay journal can name — not an import choice. Each +variant carries its backend's seam (router/router-tools/bridge/cli/cli-worktree/sandbox). + +*** + +### StopDecision + +> **StopDecision** = \{ `stop`: `false`; \} \| \{ `stop`: `true`; `reason`: `string`; \} + +A stop rule's answer. `reason` is required when stopping — a run that ends must be able to say + why in the result, and an unexplained early stop is indistinguishable from a bug. + +*** + +### StopRule + +> **StopRule** = (`view`) => [`StopDecision`](#stopdecision) + +Evaluated from the progress feed, never from the budget. Pure and synchronous: it is called on + the driver's hot path, once per turn. + +#### Parameters + +##### view + +[`ProgressView`](#progressview) + +#### Returns + +[`StopDecision`](#stopdecision) + +*** + +### SupervisorProfile -> **DispatchStopReason** = `"drained"` \| `"not-admitted"` \| `"stopped"` \| `"aborted"` +> **SupervisorProfile** = `AgentProfile` -Why the dispatcher stopped admitting work. `drained` = the queue ran dry (the ordinary end); - `not-admitted` = the conserved pool or the depth ceiling refused a spawn; `stopped` = the - caller's `shouldStop` returned true; `aborted` = the scope's signal fired. +A supervisor is an ordinary, complete `AgentProfile` playing the supervisor role. + Runtime policy (budget, concurrency, analysts, stop rules) stays in `SupervisorAgentDeps`; it is + live execution state, not agent identity. An omitted harness or `cli-base` selects the in-process + router brain; every other harness is materialized by `driveHarness`. *** -### RunContext +### SupervisorNodeContextSeed -> **RunContext** = [`InMemoryRunContext`](#inmemoryruncontext) +> **SupervisorNodeContextSeed** = `Omit`\<[`SupervisorNodeContext`](#supervisornodecontext), `"nodeId"` \| `"profile"` \| `"task"`\> -The stores a supervised run needs, in-memory or file-backed. `InMemoryRunContext` is the - historical name for the same shape. +Context known before `Agent.act`; Runtime adds the concrete node, profile, and task. *** -### ExecutorConfig +### ResolveSupervisorTools -> **ExecutorConfig** = `object` & [`RouterSeam`](#routerseam) \| `object` & [`RouterToolsSeam`](#routertoolsseam) \| `object` & [`BridgeSeam`](#bridgeseam) \| `object` & [`CliSeam`](#cliseam) \| `object` & [`CliWorktreeSeam`](#cliworktreeseam) \| `object` & [`ProviderSeam`](#providerseam) \| `object` & [`PiSeam`](#piseam) \| `object` & [`SandboxSeam`](#sandboxseam) +> **ResolveSupervisorTools** = (`context`) => `ReadonlyArray`\<[`SupervisorToolDescriptor`](#supervisortooldescriptor)\> \| `Promise`\<`ReadonlyArray`\<[`SupervisorToolDescriptor`](#supervisortooldescriptor)\>\> -Config for [createExecutor](#createexecutor): the backend is DATA — the cost dial a profile, -an experiment config, or a replay journal can name — not an import choice. Each -variant carries its backend's seam (router/router-tools/bridge/cli/cli-worktree/sandbox). +Product policy for the tools one exact supervisor node may call. Resolved once per node. -*** +#### Parameters -### StopDecision +##### context -> **StopDecision** = \{ `stop`: `false`; \} \| \{ `stop`: `true`; `reason`: `string`; \} +[`SupervisorNodeContext`](#supervisornodecontext) -A stop rule's answer. `reason` is required when stopping — a run that ends must be able to say - why in the result, and an unexplained early stop is indistinguishable from a bug. +#### Returns + +`ReadonlyArray`\<[`SupervisorToolDescriptor`](#supervisortooldescriptor)\> \| `Promise`\<`ReadonlyArray`\<[`SupervisorToolDescriptor`](#supervisortooldescriptor)\>\> *** -### StopRule +### ObserveSupervisorNodeEvent -> **StopRule** = (`view`) => [`StopDecision`](#stopdecision) +> **ObserveSupervisorNodeEvent** = (`context`, `event`, `record`) => `void` \| `Promise`\<`void`\> -Evaluated from the progress feed, never from the budget. Pure and synchronous: it is called on - the driver's hot path, once per turn. +Context-aware observer used internally to bind product transactions to the actual live node. #### Parameters -##### view +##### context -[`ProgressView`](#progressview) +[`SupervisorNodeContext`](#supervisornodecontext) + +##### event + +[`CoordinationEvent`](index.md#coordinationevent) + +##### record + +[`BusRecord`](#busrecord)\<[`CoordinationEvent`](index.md#coordinationevent)\> #### Returns -[`StopDecision`](#stopdecision) +`void` \| `Promise`\<`void`\> *** @@ -16042,9 +17125,9 @@ Evaluated from the progress feed, never from the budget. Pure and synchronous: i > **DriveHarness** = (`args`) => `Promise`\<`void`\> -How to run a sandboxed harness as the DRIVER, with the coordination verbs mounted — the substrate +How to run an external harness as the DRIVER, with the coordination verbs mounted — the substrate seam the caller supplies (mirrors `makeWorkerAgent` for spawned children). It runs `profile` on - `task` in its backend (sandbox / cli-bridge) with `coordinationMcpUrl` mounted as an MCP server, + `task` in its backend (remote sandbox or local CLI bridge) with `coordinationMcpUrl` mounted as an MCP server, so the harness calls spawn_agent / await_event / stop as native tools over the live scope. #### Parameters @@ -16067,6 +17150,13 @@ How to run a sandboxed harness as the DRIVER, with the coordination verbs mounte `string` +###### coordinationTools + +`ReadonlyArray`\<`Omit`\<[`McpToolDescriptor`](mcp.md#mcptooldescriptor), `"handler"`\>\> + +Data-only product tool surface mounted on the coordination MCP. Runtime-owned drivers include + this in their materialization evidence without persisting executable handlers. + #### Returns `Promise`\<`void`\> @@ -16092,6 +17182,75 @@ External executors can register additional runtime strings without widening this *** +### MaterializedModelIdentity + +> **MaterializedModelIdentity** = \{ `status`: `"known"`; `id`: `string`; \} \| \{ `status`: `"unknown"`; `reason`: `string`; \} + +A named model carried into an execution, or an explicit reason the exact model is unknowable. + +*** + +### UnknownMaterializationReason + +> **UnknownMaterializationReason** = `"executor-did-not-report"` \| `"invalid-executor-report"` \| `"root-agent-did-not-report"` + +Why exact materialization evidence is unavailable for a node. + +*** + +### ProfileMaterializationReceipt + +> **ProfileMaterializationReceipt** = \{ `status`: `"known"`; `authoredProfileDigest`: `Sha256Digest`; `effectiveProfileDigest`: `Sha256Digest`; `materializationPlanDigest`: `Sha256Digest`; `platformAttachmentsDigest?`: `Sha256Digest`; `runtime`: [`Runtime`](#runtime-2); `backend`: `string`; `model`: [`MaterializedModelIdentity`](#materializedmodelidentity); `execution`: [`MaterializedExecutionIdentity`](#materializedexecutionidentity); `materializer`: `string`; \} \| \{ `status`: `"unknown"`; `authoredProfileDigest?`: `Sha256Digest`; `runtime`: [`Runtime`](#runtime-2); `reason`: [`UnknownMaterializationReason`](#unknownmaterializationreason); \} + +What the kernel can prove about one node's actual execution plan. + +*** + +### ExecutionBindingReceipt + +> **ExecutionBindingReceipt** = \{ `status`: `"known"`; `attemptId`: `string`; `materializationReceiptDigest`: `Sha256Digest`; `bindingDigest`: `Sha256Digest`; `descriptor`: `Readonly`\<`Record`\<`string`, `string` \| `number` \| `boolean` \| `null`\>\>; \} \| \{ `status`: `"unknown"`; `attemptId`: `string`; `materializationReceiptDigest`: `Sha256Digest`; `reason`: [`UnknownMaterializationReason`](#unknownmaterializationreason); \} + +One attempt's immutable link from a stable materialization plan to its actual transport. + +*** + +### RootMaterialization + +> **RootMaterialization** = \{ `runtime`: [`Runtime`](#runtime-2); `declaration`: [`ExecutorMaterialization`](#executormaterialization); `binding`: `Omit`\<[`ExecutorExecutionBinding`](#executorexecutionbinding), `"attemptId"`\>; \} \| \{ `runtime`: [`Runtime`](#runtime-2); `declaration`: `"deferred"`; `authoredProfile`: `AgentProfile`; \} + +Trusted root composition evidence. Generic `Agent.act` roots omit this and remain unknown. + +#### Union Members + +##### Type Literal + +\{ `runtime`: [`Runtime`](#runtime-2); `declaration`: [`ExecutorMaterialization`](#executormaterialization); `binding`: `Omit`\<[`ExecutorExecutionBinding`](#executorexecutionbinding), `"attemptId"`\>; \} + +*** + +##### Type Literal + +\{ `runtime`: [`Runtime`](#runtime-2); `declaration`: `"deferred"`; `authoredProfile`: `AgentProfile`; \} + +###### runtime + +> `readonly` **runtime**: [`Runtime`](#runtime-2) + +The runtime-owned external adapter will publish the exact declaration after its dynamic +platform attachment (for example a coordination URL) exists and before paid work starts. + +###### declaration + +> `readonly` **declaration**: `"deferred"` + +###### authoredProfile + +> `readonly` **authoredProfile**: `AgentProfile` + +Exact admitted profile used to validate the stable effective identity at publication. + +*** + ### ExecutorFactory > **ExecutorFactory**\<`Out`\> = (`spec`, `ctx`) => [`Executor`](index.md#executor-2)\<`Out`\> @@ -16152,16 +17311,17 @@ Deterministic node id — `${parent}:s${seq}` from the cursor order, never wall- ### SpawnRejection -> **SpawnRejection** = `"budget-exhausted"` \| `"depth-exceeded"` \| `"duplicate-key"` +> **SpawnRejection** = `"budget-exhausted"` \| `"depth-exceeded"` \| `"duplicate-key"` \| `"invalid-identity"` \| `"key-conflict"` \| `"max-live-workers"` \| `"scope-aborted"` -Fail-closed spawn rejections: an exhausted pool, an exceeded recursion ceiling, or a `key` - that is still LIVE in this scope (the same assignment may not run twice concurrently). +Fail-closed spawn rejections: an exhausted pool, an exceeded recursion ceiling, a full + tree-wide worker allocation, or a `key` that is still LIVE in this scope (the same assignment + may not run twice concurrently). *** ### SpawnPrior -> **SpawnPrior**\<`Out`\> = \{ `state`: `"completed"`; `settled`: [`Settled`](index.md#settled)\<`Out`\> & `object`; \} \| \{ `state`: `"retried"`; `priorId`: [`NodeId`](#nodeid-1); `reason`: `string`; \} \| \{ `state`: `"lost"`; `priorId`: [`NodeId`](#nodeid-1); \} +> **SpawnPrior**\<`Out`\> = \{ `state`: `"completed"`; `settled`: [`Settled`](index.md#settled)\<`Out`\> & `object`; \} \| \{ `state`: `"retried"`; `priorId`: [`NodeId`](#nodeid-3); `reason`: `string`; \} \| \{ `state`: `"lost"`; `priorId`: [`NodeId`](#nodeid-3); \} What a KEYED spawn resolved to when the key had a prior attempt. Absent on a fresh key (and on every unkeyed spawn). `'completed'` is the exactly-once path: NOTHING was spawned — the handle @@ -16169,8 +17329,10 @@ references the prior settled node and `settled` is the committed result. `'retri `'lost'` DID spawn fresh: the prior attempt settled `down` (retried) or was journaled as started but never settled — the process died with it in flight and the built-in executors cannot re-attach to a dead process's work, so the result is explicitly in doubt (lost), never -silently duplicated. An executor that CAN re-attach to a still-running external execution (a -live sandbox box) extends this union with an adoption state; none of the built-ins can today. +silently duplicated. On restart, an in-doubt attempt's full declared reservation is charged and +its telemetry remains unknown; a fresh retry is admitted only from safely remaining capacity. +An executor that CAN re-attach to a still-running external execution extends this union with an +adoption state; none of the built-ins can today. #### Type Parameters @@ -16182,7 +17344,7 @@ live sandbox box) extends this union with an adoption state; none of the built-i ### SpawnEvent -> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-1); `parent?`: [`NodeId`](#nodeid-1); `label`: `string`; `key?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-2); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-1); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-1); `reason`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-1); `parent?`: [`NodeId`](#nodeid-1); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-1); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-1); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} +> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-3); `parent?`: [`NodeId`](#nodeid-3); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-2); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-3); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-3); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-3); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-3); `reason`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-3); `parent?`: [`NodeId`](#nodeid-3); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-3); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-3); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO timestamp for human inspection only (NOT a replay input). @@ -16191,7 +17353,7 @@ Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO ##### Type Literal -\{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-1); `parent?`: [`NodeId`](#nodeid-1); `label`: `string`; `key?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-2); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-3); `parent?`: [`NodeId`](#nodeid-3); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-2); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} ###### kind @@ -16199,11 +17361,11 @@ Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO ###### id -> **id**: [`NodeId`](#nodeid-1) +> **id**: [`NodeId`](#nodeid-3) ###### parent? -> `optional` **parent?**: [`NodeId`](#nodeid-1) +> `optional` **parent?**: [`NodeId`](#nodeid-3) ###### label @@ -16216,6 +17378,12 @@ Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO The semantic spawn key (`SpawnOpts.key`), when the spawn carried one — what a resumed run matches to resolve the same assignment to its committed result. +###### assignmentId? + +> `optional` **assignmentId?**: `string` + +Manager-scoped assignment identity used to join unkeyed and keyed work alike. + ###### budget > **budget**: [`Budget`](index.md#budget-4) @@ -16224,6 +17392,69 @@ The semantic spawn key (`SpawnOpts.key`), when the spawn carried one — what a > **runtime**: [`Runtime`](#runtime-2) +###### identity? + +> `optional` **identity?**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +Exact profile/task digests plus trusted candidate/campaign attribution when available. + +###### seq + +> **seq**: `number` + +###### at + +> **at**: `string` + +*** + +##### Type Literal + +\{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-3); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} + +###### kind + +> **kind**: `"execution-bound"` + +Volatile transport/session binding for exactly one attempt. The full binding is retained +only by digest; descriptor fields are safe structural labels, never credential-bearing URLs. + +###### id + +> **id**: [`NodeId`](#nodeid-3) + +###### binding + +> **binding**: [`ExecutionBindingReceipt`](#executionbindingreceipt) + +###### seq + +> **seq**: `number` + +###### at + +> **at**: `string` + +*** + +##### Type Literal + +\{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-3); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} + +###### kind + +> **kind**: `"materialized"` + +Trusted runtime transformation from the authorized profile to actual wire bytes. + +###### id + +> **id**: [`NodeId`](#nodeid-3) + +###### receipt + +> **receipt**: [`ProfileMaterializationReceipt`](#profilematerializationreceipt) + ###### seq > **seq**: `number` @@ -16236,7 +17467,7 @@ The semantic spawn key (`SpawnOpts.key`), when the spawn carried one — what a ##### Type Literal -\{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-1); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-3); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} ###### kind @@ -16244,7 +17475,7 @@ The semantic spawn key (`SpawnOpts.key`), when the spawn carried one — what a ###### id -> **id**: [`NodeId`](#nodeid-1) +> **id**: [`NodeId`](#nodeid-3) ###### status @@ -16268,6 +17499,19 @@ Content-addressed result pointer; rehydrates `out` from `ResultBlobStore`. > `optional` **infra?**: `boolean` +###### reason? + +> `optional` **reason?**: `string` + +Exact child failure. Present on every new `status: 'down'` record; optional only so +journals written before this field existed remain replayable. + +###### trace? + +> `optional` **trace?**: [`WorkerTraceEvidence`](index.md#workertraceevidence) + +Structured tool evidence. Optional only for journals written before trace capture. + ###### seq > **seq**: `number` @@ -16280,13 +17524,13 @@ Content-addressed result pointer; rehydrates `out` from `ResultBlobStore`. ##### Type Literal -\{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-1); `reason`: `string`; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-3); `reason`: `string`; `seq`: `number`; `at`: `string`; \} *** ##### Type Literal -\{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-1); `parent?`: [`NodeId`](#nodeid-1); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-3); `parent?`: [`NodeId`](#nodeid-3); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} ###### kind @@ -16299,11 +17543,11 @@ A wait-state node was ARMED. Lives in the SPAWN-ORDINAL namespace (`seq` is the ###### id -> **id**: [`NodeId`](#nodeid-1) +> **id**: [`NodeId`](#nodeid-3) ###### parent? -> `optional` **parent?**: [`NodeId`](#nodeid-1) +> `optional` **parent?**: [`NodeId`](#nodeid-3) ###### label @@ -16329,7 +17573,7 @@ A wait-state node was ARMED. Lives in the SPAWN-ORDINAL namespace (`seq` is the ##### Type Literal -\{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-1); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-3); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} ###### kind @@ -16342,7 +17586,7 @@ A wait-state node SETTLED — the cursor-namespace twin of `settled`, kept disti ###### id -> **id**: [`NodeId`](#nodeid-1) +> **id**: [`NodeId`](#nodeid-3) ###### by @@ -16364,7 +17608,7 @@ A wait-state node SETTLED — the cursor-namespace twin of `settled`, kept disti ##### Type Literal -\{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-1); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-3); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} ###### kind @@ -16380,7 +17624,7 @@ A driver's OWN inference spend, journaled separately from spawned-child work — ###### id -> **id**: [`NodeId`](#nodeid-1) +> **id**: [`NodeId`](#nodeid-3) ###### spend @@ -16870,19 +18114,30 @@ the other CLI leaves; the authored systemPrompt + model reach the harness via § Ceiling on continuation turns. Turn 0 is the task; every later turn is a folded steer, so this bounds how many times a supervisor may redirect ONE worker before it must respawn. +*** + +### DEFAULT\_AUTHORED\_PROFILE\_SECURITY\_POLICY + +> `const` **DEFAULT\_AUTHORED\_PROFILE\_SECURITY\_POLICY**: `AgentProfileSecurityPolicy` + +Manager-authored profiles are untrusted until product policy says otherwise. Remote MCP and +ambient connection grants therefore fail closed by default, in addition to local MCP and hooks. + +*** + +### WORKER\_TOOL\_TRACE\_SCHEMA\_VERSION + +> `const` **WORKER\_TOOL\_TRACE\_SCHEMA\_VERSION**: `1` + +Schema version for content-addressed worker tool-trace artifacts. + ## Functions ### contentAddress() > **contentAddress**(`artifact`): `string` -Mint the content-addressed `outRef` for a result artifact: `sha256:` over a -stable JSON encoding. Producers call this to derive the `outRef` they journal and -`put`; the FS/in-mem stores re-derive it on `put` to verify the supplied ref -matches (fail loud on a mismatch — a forged ref breaks the replay invariant). - -Stable encoding: object keys are sorted recursively so two structurally-equal -artifacts hash identically regardless of key insertion order. +Stable content address shared by result and trace artifacts. #### Parameters @@ -17710,7 +18965,7 @@ Fail loud (no silent empty findings): ##### scope -[`Scope`](index.md#scope)\<[`Outcome`](#outcome-1)\<`D`\>\> +[`Scope`](index.md#scope)\<[`Outcome`](#outcome-2)\<`D`\>\> ##### options @@ -17787,7 +19042,7 @@ readonly `AnalystFinding`[] ##### settledSoFar -readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-1)\<`D`\>\>[] +readonly [`Settled`](index.md#settled)\<[`Outcome`](#outcome-2)\<`D`\>\>[] #### Returns @@ -18142,7 +19397,7 @@ unrunnable — refuse it at definition time, not at the first spawn. Pure; no I/ ### runPersonified() -> **runPersonified**\<`Task`, `D`\>(`options`): `Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<[`Outcome`](#outcome-1)\<`D`\>\>\> +> **runPersonified**\<`Task`, `D`\>(`options`): `Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<[`Outcome`](#outcome-2)\<`D`\>\>\> Compose the persona + chosen shape onto a fresh keystone `Supervisor`. Resolves the shape (a factory verbatim, or a registered name through `builtinShapes`), applies it to a @@ -18168,7 +19423,7 @@ default-shape fallback. #### Returns -`Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<[`Outcome`](#outcome-1)\<`D`\>\>\> +`Promise`\<[`SupervisedResult`](index.md#supervisedresult)\<[`Outcome`](#outcome-2)\<`D`\>\>\> *** @@ -19194,7 +20449,7 @@ Multi-generation strategy search: author candidates from tournament losses, play ### depthStrategy() -> **depthStrategy**(`surface`, `task`, `opts`, `cfg`): [`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> +> **depthStrategy**(`surface`, `task`, `opts`, `cfg`): [`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-2)\<`unknown`\>\> DEPTH: one persistent artifact, carried across analyst-steered shots. @@ -19220,13 +20475,13 @@ DEPTH: one persistent artifact, carried across analyst-steered shots. #### Returns -[`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> +[`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-2)\<`unknown`\>\> *** ### breadthStrategy() -> **breadthStrategy**(`_surface`, `task`, `opts`, `cfg`): [`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> +> **breadthStrategy**(`_surface`, `task`, `opts`, `cfg`): [`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-2)\<`unknown`\>\> BREADTH: K independent rollouts (each own artifact), verifier picks the best. @@ -19252,7 +20507,7 @@ BREADTH: K independent rollouts (each own artifact), verifier picks the best. #### Returns -[`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> +[`Agent`](#agent-1)\<`unknown`, [`Outcome`](#outcome-2)\<`unknown`\>\> *** @@ -19666,7 +20921,7 @@ Drive a team of agents (spawned + steered by `profile`) to solve a graded `Agent ##### profile -[`SupervisorProfile`](#supervisorprofile) +`AgentProfile` ##### task @@ -19726,8 +20981,8 @@ The supervisor SKILL — the how-to the supervisor reads (its system prompt). TH > **authoredWorker**(`profile`, `opts`): [`Agent`](#agent-1)\<`unknown`, `unknown`\> -Build a worker AGENT from a profile the supervisor authored: the authored `systemPrompt` + - `model` shape the worker's one model call; the deliverable gates settlement (valid ⟺ delivered). +Build a router-only worker from an authored profile. This helper executes the prompt/model axes; + use `workerFromBackend` for full materialization of tools, MCP, resources, hooks, and subagents. #### Parameters @@ -19840,12 +21095,13 @@ Fold a normalized `UsageEvent` array into a `Spend`. Tokens and usd are separate ### createBudgetPool() -> **createBudgetPool**(`root`, `now?`): [`BudgetPool`](#budgetpool) +> **createBudgetPool**(`root`, `now?`, `restore?`): [`BudgetPool`](#budgetpool) Create a conserved reservation pool from a root `Budget`. `now()` is injected so the deadline readout is deterministic; defaults to `Date.now` for non-test callers. The -absolute deadline is fixed at construction (`now() + budget.deadlineMs`) so the -readout's `deadlineMs` is a stable wall-clock instant, not a shrinking remainder. +absolute deadline for a fresh pool is fixed at construction (`now() + budget.deadlineMs`). A +restored pool instead retains `restore.absoluteDeadlineMs`, so restart never slides the original +wall-clock limit. The readout is an absolute instant, not a shrinking remainder. #### Parameters @@ -19857,6 +21113,10 @@ readout's `deadlineMs` is a stable wall-clock instant, not a shrinking remainder () => `number` +##### restore? + +[`BudgetPoolRestore`](#budgetpoolrestore) = `{}` + #### Returns [`BudgetPool`](#budgetpool) @@ -19964,6 +21224,10 @@ Stand up the coordination MCP over a live scope. The HOST address is `127.0.0.1` [`MakeWorkerAgent`](#makeworkeragent) +###### authorizeDownMessage? + +[`AuthorizeDownMessage`](#authorizedownmessage) + ###### perWorker [`Budget`](index.md#budget-4) @@ -20017,9 +21281,16 @@ Idle time after which `observe_agent` reports a worker as stalled. ###### onEvent? -(`event`) => `void` \| `Promise`\<`void`\> +(`event`, `record`) => `void` \| `Promise`\<`void`\> + +Pass-through subscriber for every bus event, including pre-delivery instruction receipts and +steer/answer delivery outcomes. -Pass-through subscriber for every bus event (settled / question / finding). +###### replaySettlements? + +`boolean` + +Re-publish resume-time settlements through the awaited observer before this server listens. ###### questionPolicy? @@ -20031,6 +21302,13 @@ readonly [`QuestionRecord`](mcp.md#questionrecord)[] Questions replayed from a prior process of this run — seeds the question ledger. +###### nodeTools? + +readonly [`McpToolDescriptor`](mcp.md#mcptooldescriptor)[] + +Product-selected tools already bound to this exact supervisor node. They share this server + with the coordination verbs, so the existing MCP duplicate-name guard applies before listen. + #### Returns `Promise`\<[`CoordinationMcpHandle`](#coordinationmcphandle)\> @@ -20328,7 +21606,7 @@ readonly [`FinalizerSettled`](#finalizersettled)[] ###### budget -`Readonly`\<\{ `tokensLeft`: `number`; `usdLeft`: `number`; `usdCapped`: `boolean`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +`Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> #### Returns @@ -20372,6 +21650,29 @@ readonly `string`[] \| `undefined` *** +### assertProfileModelsAllowed() + +> **assertProfileModelsAllowed**(`profile`, `allowed`): `void` + +Check every canonical model-bearing field in a complete profile, including the models a +backend may select for cheap work, named subagents, or modes. + +#### Parameters + +##### profile + +`AgentProfile` + +##### allowed + +readonly `string`[] \| `undefined` + +#### Returns + +`void` + +*** + ### patchDelivered() > **patchDelivered**(`options?`): [`DeliverableSpec`](#deliverablespec)\<[`WorktreeHarnessResult`](#worktreeharnessresult)\> @@ -20473,8 +21774,9 @@ resumes when it is re-run with the SAME `runId` and the SAME `dir`: the committe back on `Scope.resume` (rehydrated by `replaySpawnTree`) instead of being re-executed. Layout: `${dir}/spawn-journal.jsonl` (one JSONL record per event), `${dir}/blobs/` (one -content-addressed JSON file per settled result), and `${dir}/coordination-log.jsonl` (questions -+ findings, replayed into a resumed driver). The directory is created on first write. +content-addressed JSON file per settled result), and `${dir}/coordination-log.jsonl` +(questions, findings, answer decisions, and authorized continuation receipts retained as +evidence). The directory is created on first write. Opt-in by construction — `createInMemoryRunContext()` is unchanged and stays the default, so no existing consumer writes to disk or resumes unless it asks for this. @@ -20528,8 +21830,8 @@ factory)` for any additional runtime — and a BYO `AgentSpec.executor` resolves without touching the registry at all. NOT a closed switch; registration + BYO ARE the extension points. -`resolve` precedence (frozen in `ExecutorRegistry`): a BYO `spec.executor` → -`harness === null` → the `'router'` factory; else a registered factory for the +`resolve` precedence (frozen in `ExecutorRegistry`): a BYO `spec.executorFactory` → +`spec.executor` → `harness === null` → the `'router'` factory; else a registered factory for the harness-derived runtime (`'sandbox'` for any `BackendType`); else fail loud. #### Returns @@ -20792,7 +22094,7 @@ One-call supervisor: build + run a supervisor from its profile with sensible def ##### profile -[`SupervisorProfile`](#supervisorprofile) +`AgentProfile` ##### task @@ -20818,7 +22120,7 @@ Build a supervisor `Agent` from its profile: the brain resolves from `profile.ha ##### profile -[`SupervisorProfile`](#supervisorprofile) +`AgentProfile` ##### deps @@ -20852,6 +22154,76 @@ Create a supervisor that owns one recursive agent execution tree. *** +### captureWorkerTraceEvidence() + +> **captureWorkerTraceEvidence**(`readSource`, `blobs`, `executed`): `Promise`\<[`WorkerTraceEvidence`](index.md#workertraceevidence)\> + +Collect and persist one executor's structured tool trace without changing its task outcome. + +#### Parameters + +##### readSource + +(() => [`TraceSource`](#tracesource-1) \| `undefined`) \| `undefined` + +##### blobs + +[`ResultBlobStore`](#resultblobstore) + +##### executed + +`boolean` + +#### Returns + +`Promise`\<[`WorkerTraceEvidence`](index.md#workertraceevidence)\> + +*** + +### workerTraceAnalysisStore() + +> **workerTraceAnalysisStore**(`evidence`, `blobs`): `Promise`\<`TraceAnalysisStore`\> + +Rehydrate exact persisted spans through agent-eval's one bounded trace-analysis adapter. + +#### Parameters + +##### evidence + +[`WorkerTraceEvidence`](index.md#workertraceevidence) + +##### blobs + +`Pick`\<[`ResultBlobStore`](#resultblobstore), `"get"`\> + +#### Returns + +`Promise`\<`TraceAnalysisStore`\> + +*** + +### parseWorkerToolTraceArtifact() + +> **parseWorkerToolTraceArtifact**(`value`, `traceRef?`): [`WorkerToolTraceArtifact`](#workertooltraceartifact) + +Validate a stored trace artifact before an analyst or replay trusts it. + +#### Parameters + +##### value + +`unknown` + +##### traceRef? + +`string` = `''` + +#### Returns + +[`WorkerToolTraceArtifact`](#workertooltraceartifact) + +*** + ### decodeToolPart() > **decodeToolPart**(`part`, `harness?`): [`ToolStepInput`](#toolstepinput) \| `undefined` @@ -21127,8 +22499,9 @@ Structural validation, independent of the run. Returns null when the spec is usa Build a worktree-CLI leaf `Executor`. Per-spawn (a fresh worktree + abort + teardown each), so a fanout of N profiles = N parallel worktrees that never clobber each other. -Fail-loud: an empty `repoRoot`/`harness`/`taskPrompt` throws at construction. `resultArtifact()` -before `execute()` resolves throws. +Fail-loud: an empty `repoRoot`/`harness` or an explicitly empty `taskPrompt` throws at +construction. Calling `execute(undefined, signal)` without a configured prompt throws before a +worktree is created. `resultArtifact()` before `execute()` resolves throws. #### Parameters @@ -21462,6 +22835,18 @@ Re-exports [Supervisor](index.md#supervisor) *** +### WorkerTraceEvidence + +Re-exports [WorkerTraceEvidence](index.md#workertraceevidence) + +*** + +### WorkerTraceUnavailableReason + +Re-exports [WorkerTraceUnavailableReason](index.md#workertraceunavailablereason) + +*** + ### Driver Re-exports [Driver](index.md#driver) diff --git a/docs/architecture.md b/docs/architecture.md index 8ebeca7e..313b3998 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -357,13 +357,12 @@ A **leaf** is an `act` that returns without touching `scope`. A **driver** is an that spawns children and reacts to them. Same type — the role is behavior, not a class (the full prose is §1). -The `Scope` it runs inside is **5 verbs** (`types.ts`) — a budget-conserving reactive -nursery: +The `Scope` it runs inside is the budget-conserving reactive control surface (`types.ts`): ``` scope ───────────────────────────────────────────────────────────────────────────────────── │ - ├─ spawn(agent, task, {budget,label}) → {ok,handle} | {ok:false, 'budget-exhausted'|'depth-exceeded'} + ├─ spawn(agent, task, {budget,label,key?}) → {ok,handle,prior?} | {ok:false, SpawnRejection} │ reserves budget ATOMICALLY from a conserved pool, fail-closed ⟸ THE equal-compute invariant │ ├─ next() → Promise the WAKE cursor: resolves as each child settles, in seq order @@ -372,20 +371,22 @@ nursery: ├─ send(nodeId, msg) → bool STEER a running child (next-instruction / interrupt) │ in-process = direct call · across a sandbox = the SAME verb as an MCP tool │ - ├─ view → TreeView the live tree (in-memory, O(live)) — what the topology viewer renders - └─ budget → {tokensLeft, usdLeft, deadlineMs, reservedTokens} + ├─ wait(spec) durable timer or named-predicate wait + ├─ progress(nodeId) / traceSource(nodeId) explicit live observation + ├─ meter(spend) / recordMaterialization(receipt) runtime accounting and wire-profile evidence + ├─ view / workerCapacity live tree and shared execution slots + └─ budget → {tokensLeft,tokensKnown,usdLeft,usdKnown,iterationsLeft,deadlineMs,reservedTokens} ``` Two facts make this the whole game: -- `spawn` **reserves** from a shared pool and refunds the unspent remainder on settle, so - `Σk(treatment) ≡ Σk(blind)` by construction — no arm can buy more compute - (`supervise/budget.ts`). -- `next()` is the *only* way to observe a child, so a driver reacts to **settlements**, - never reaches inside a child. +- `spawn` **reserves** from one root total and refunds the unspent remainder on settle. + A nested driver partitions only its reserved allocation, then reconciles the whole subtree once, so `Σk(treatment) ≡ Σk(blind)` by construction — no arm can buy more compute (`supervise/budget.ts`). +- `next()` is the only path that consumes a child's terminal result. + Live observation is explicit and read-only through `progress` and `traceSource`; neither can manufacture a settlement. -The ask/answer edges of the question/command hierarchy are **built** — `ask_parent` up -and `answer_question` down (`src/mcp/tools/coordination.ts:159-160`), priority-queued on -the event bus; salience filtering and the cross-box durable mailbox are not. See **§13.6**. +The ask/answer edges of the question/command hierarchy are **built** — `ask_parent` up and `answer_question` down (`src/mcp/tools/coordination.ts`), priority-queued on the event bus. +Every steer/answer authorization receipt is committed before delivery and retained as restart evidence, but Runtime never auto-delivers that old instruction to a replacement worker. +Salience filtering and the cross-box durable mailbox are not built; see **§13.6**. ### 13.2 The tree — drivers of drivers, one recursive atom @@ -418,12 +419,9 @@ the event bus; salience filtering and the cross-box durable mailbox are not. See → equal-compute holds at EVERY depth (`supervise/budget.ts`) ``` -- **REAL** — one recursive `Agent` node, not two types: `Agent.act(task, scope)` in - `src/runtime/supervise/types.ts:49`. The roles are the *same* atom; a node is a - "driver" only because its tools spawn children. A child whose `act` calls - `scope.spawn` is a driver too, with its **own sub-scope** (depth+1, bounded by - `maxDepth` + the *same* pool) — recursion isn't a feature, it's the absence of a - base case (`supervise/supervisor.ts`, `supervise/scope.ts`). +- **REAL** — one recursive `Agent` node, not two types: `Agent.act(task, scope)` in `src/runtime/supervise/types.ts:49`. + The roles are the *same* atom; a node is a "driver" only because its tools spawn children. + A child whose `act` calls `scope.spawn` is a driver too, with its **own sub-scope** (depth+1, bounded by `maxDepth` + a partition of the same root total) — recursion isn't a feature, it's the absence of a base case (`supervise/supervisor.ts`, `supervise/scope.ts`). - **REAL** — the **leaf** at the bottom is where a real coding harness runs, opaque and self-parallelizing internally; the `runAgentRounds` kernel (`src/runtime/run-loop.ts`) is composed as one leaf execution backend. Everything above it is the same `act`/`Scope` @@ -437,12 +435,17 @@ the event bus; salience filtering and the cross-box durable mailbox are not. See and `canonical-api.md` §1.5): a supervisor's intelligence is *writing full AgentProfiles for its children*. The coordination toolbox `spawn_agent` carries the child profile (`src/mcp/tools/coordination.ts`). -- The in-process driver brain is `driverAgent` - (`supervise/coordination-driver.ts`) running the owned tool-loop executor - `routerToolsInlineExecutor` (`supervise/runtime.ts`). A driver/supervisor's brain is - driven from its `AgentProfile` (tools = the coordination verbs); inferring the brain - entirely from the profile so a driver is *just* a profile with zero special cases is - not yet wired end-to-end. +- The in-process driver brain is `driverAgent` (`supervise/coordination-driver.ts`) running the owned tool-loop executor `routerToolsInlineExecutor` (`supervise/runtime.ts`). + A driver/supervisor's brain is driven from its `AgentProfile`: prompt + model for the deliberately narrow in-process router arm, or the complete materialized profile for an external-harness arm. +- **REAL** — `supervise(profile, task, { backend })` validates and freezes every authored child profile before budget reservation, applies shared security plus optional product authorization, and preserves the authorized profile through execution (`supervise/supervise.ts`). + A child marked `metadata.role: 'driver'` recursively becomes another supervisor over the same budget; every other child resolves to a leaf. +- **REAL** — a local external-harness supervisor runs automatically through a `bridge` `driverBackend ?? backend` with the live coordination MCP injected under one reserved alias. + Its own tools, resources, MCP servers, hooks, subagents, permissions, modes, prompt, and model remain profile data sent to that backend (`supervise/supervise.ts`, `supervise/runtime.ts`). + A pre-execution `materialized` journal event binds the authored-profile, effective-profile, and platform-attachment digests to the node that ran. +- **LIMIT** — a remote sandbox cannot reach the loopback coordination server automatically. + It needs an explicit `driveHarness` that provides a reachable relay or tunnel. +- **LIMIT** — the in-process router arm has no environment in which to materialize profile resources, hooks, subagents, permissions, or modes. + It executes prompt + model and uses explicit `extraTools`; choose an external backend when the other profile axes must run. ### 13.3 The within-run self-improvement loop (§1's agent-driver, drawn) @@ -556,14 +559,15 @@ not agent-to-agent messaging. **Built** (`src/mcp/tools/coordination.ts`, `src/runtime/supervise/event-bus.ts`, `src/runtime/supervise/inbox.ts`): -- `ask_parent` up + `answer_question` down (`src/mcp/tools/coordination.ts:159-160`) — +- `ask_parent` up + `answer_question` down (`src/mcp/tools/coordination.ts`) — a blocking question rides the ONE typed pipe, **priority-queued** ahead of queued settles/findings (the event bus); the answer routes down to the child's inbox. -- `steer_worker` — the down-leg for any live worker (instruction / correction / +- `steer_agent` — the down-leg for any live worker (instruction / correction / continuation); queued messages flush at step boundaries AND before the worker may - settle; a forceful `steer_worker({interrupt:true})` aborts the in-flight turn (the + settle; a forceful `steer_agent({interrupt:true})` aborts the in-flight turn (the inbox). -- `notify` up — every settle/decision is teed upward on the lifecycle hook stream. +- `agent.spawn` / `agent.child` lifecycle events — every spawn and consumed settlement is + sent to the runtime hook stream. **Not built:** the **salience tag** on decisions (so the top doesn't drown), the cross-box durable mailbox (§13.9), budget-pause-while-awaiting. @@ -648,17 +652,23 @@ within-run column splits into in-flight and across-round). with the three timescales as internal composition — so "are we improving skills in the loop?" has one place to look — is not yet wired. -### 13.9 Durability — by design, not yet end-to-end +### 13.9 Durability — exact at each implemented boundary ``` - same box : in-process queue ── REAL (tested) - cross box : durable mailbox on the parent's box ── designed (the interface is ready) + same process : in-process event queue ── REAL (tested) + same host : file journal + blobs + coordination log across restart ── REAL (tested) + cross box : durable mailbox on the parent's box ── designed ``` - **REAL** — the event bus is transport-agnostic *on purpose*: same box → the in-process queue; cross box → the SAME publish/pull/subscribe surface backed by a durable mailbox on the parent's box (`supervise/event-bus.ts`). The data structure is already shaped for durability. +- **REAL** — `supervise(..., { runDir, runId })` restores committed settlements, exact profile/task/candidate identity, measured spend, pending waits, and the original absolute deadline from the file-backed stores. + The coordination log restores prior questions, findings, and authorized instruction receipts; the in-process router receives all three in its resume brief, while an external manager receives prior questions and the other evidence remains in the durable log. + An active child that lacks a terminal record is charged at its full declared reservation and its token/dollar telemetry remains explicitly unknown; a retry can use only safely remaining capacity. + Authorized instruction receipts are evidence and are never auto-delivered to a new worker. +- **LIMIT** — built-in executors do not reattach work that was active when their process died. - **designed, not built** — the cross-box (distributed-sandbox) durable binding: in-process is real and tested, the cross-box transport is the thin unbuilt part, so the up-flow can survive across distributed boxes and restarts. diff --git a/examples/ablation-suite/ablation.ts b/examples/ablation-suite/ablation.ts index 4e547745..90b5372a 100644 --- a/examples/ablation-suite/ablation.ts +++ b/examples/ablation-suite/ablation.ts @@ -209,32 +209,38 @@ export async function runAblation(opts: { // pool, with the analyst up-leg on. `superviseSurface` reports the deployable outcome + // the FULL conserved spend (driver inference + all worker work: $, tokens, latency). `shots` // stays 0 — a multi-worker supervised run has no single refine-shot count (N/A, not a real zero). - const sup = await superviseSurface({ name: 'driver', systemPrompt: driverPrompt }, t, { - surface: counter, - worker: { - routerBaseUrl: opts.worker.routerBaseUrl, - routerKey: opts.worker.routerKey, - model: opts.worker.model, - ...(opts.worker.maxTokens !== undefined ? { maxTokens: opts.worker.maxTokens } : {}), - ...(opts.worker.innerTurns !== undefined - ? { innerTurns: opts.worker.innerTurns } - : {}), - budget: arm.knobs.budget, - }, - budget: { - // Pool for the driver's turns PLUS several worker spawns (each reserves ~innerTurns+2 - // iterations) so the spawn-targeted-worker loop runs, not stall after one. The autopsy - // measures the real cost; this is intentionally not equal-k. - maxIterations: arm.knobs.budget * ((opts.worker.innerTurns ?? 6) + 2) + 16, - maxTokens: (opts.worker.maxTokens ?? 4000) * Math.max(4, arm.knobs.budget * 3), + const sup = await superviseSurface( + { name: 'driver', prompt: { systemPrompt: driverPrompt } }, + t, + { + surface: counter, + worker: { + routerBaseUrl: opts.worker.routerBaseUrl, + routerKey: opts.worker.routerKey, + model: opts.worker.model, + ...(opts.worker.maxTokens !== undefined + ? { maxTokens: opts.worker.maxTokens } + : {}), + ...(opts.worker.innerTurns !== undefined + ? { innerTurns: opts.worker.innerTurns } + : {}), + budget: arm.knobs.budget, + }, + budget: { + // Pool for the driver's turns PLUS several worker spawns (each reserves ~innerTurns+2 + // iterations) so the spawn-targeted-worker loop runs, not stall after one. The autopsy + // measures the real cost; this is intentionally not equal-k. + maxIterations: arm.knobs.budget * ((opts.worker.innerTurns ?? 6) + 2) + 16, + maxTokens: (opts.worker.maxTokens ?? 4000) * Math.max(4, arm.knobs.budget * 3), + }, + router: { + routerBaseUrl: supervisorRouter.baseUrl, + routerKey: supervisorRouter.apiKey, + model: supervisorRouter.model, + }, + analysts: failuresAnalyst(), }, - router: { - routerBaseUrl: supervisorRouter.baseUrl, - routerKey: supervisorRouter.apiKey, - model: supervisorRouter.model, - }, - analysts: failuresAnalyst(), - }) + ) if (sup.resolved) resolved++ scoreSum += sup.score perTask.push(sup.resolved ? 1 : 0) diff --git a/examples/ablation-suite/gepa-driver-prompt.ts b/examples/ablation-suite/gepa-driver-prompt.ts index fddc8b59..d02bb7df 100644 --- a/examples/ablation-suite/gepa-driver-prompt.ts +++ b/examples/ablation-suite/gepa-driver-prompt.ts @@ -130,7 +130,7 @@ export async function optimizeDriverPrompt(opts: { model: supervisorRouter.model, signal: ctx.signal, execute: () => - superviseSurface({ name: 'driver', systemPrompt }, scenario.task, { + superviseSurface({ name: 'driver', prompt: { systemPrompt } }, scenario.task, { surface, worker, // A small conserved pool: enough for the driver's turns plus several worker spawns so the diff --git a/examples/supervise/supervise.ts b/examples/supervise/supervise.ts index b0e7ebeb..fa0eace9 100644 --- a/examples/supervise/supervise.ts +++ b/examples/supervise/supervise.ts @@ -32,18 +32,20 @@ async function main(): Promise { const result = await supervise( { name: 'supervisor', - harness: null, // router brain (the supervisor reasons spawn/await/stop over the router's tool-calling) + harness: 'cli-base', // in-process router brain (the supervisor calls spawn/await/stop) // This demo overrides the shipped `defaultSupervisorPrompt` on purpose: the default tells a // supervisor to do SMALL work itself, but this supervisor has no work tools and the completion // oracle only credits a DELIVERED child — so we force the delegation path the example teaches. // Real supervisors with work tools want the default (do-small-work-yourself / spawn-when-large). - systemPrompt: - 'You are a supervisor. Produce the deliverable by delegating:\n' + - '1. Call spawn_agent with a worker profile and the task.\n' + - '2. Then call await_event and WAIT for that worker to settle — never call stop while a ' + - 'worker is still running, or its result is lost.\n' + - '3. Once a worker has delivered, call stop.\n' + - "Do not answer the task yourself — only a spawned worker's output counts as delivered.", + prompt: { + systemPrompt: + 'You are a supervisor. Produce the deliverable by delegating:\n' + + '1. Call spawn_agent with a worker profile and the task.\n' + + '2. Then call await_event and WAIT for that worker to settle — never call stop while a ' + + 'worker is still running, or its result is lost.\n' + + '3. Once a worker has delivered, call stop.\n' + + "Do not answer the task yourself — only a spawned worker's output counts as delivered.", + }, }, 'Produce the exact line: READY', { diff --git a/examples/supervisor-loop/run.ts b/examples/supervisor-loop/run.ts index 6b91cabf..d5499040 100644 --- a/examples/supervisor-loop/run.ts +++ b/examples/supervisor-loop/run.ts @@ -34,10 +34,12 @@ async function main(): Promise { const result = await supervise( { name: 'supervisor', - harness: null, - systemPrompt: - 'You are a supervisor. Spawn one worker session to produce the required line, await it with ' + - 'await_event, and stop once a worker delivered (valid). Do not answer yourself.', + harness: 'cli-base', + prompt: { + systemPrompt: + 'You are a supervisor. Spawn one worker session to produce the required line, await it ' + + 'with await_event, and stop once a worker delivered (valid). Do not answer yourself.', + }, }, demoGoal, { diff --git a/examples/supervisor-loop/shared.ts b/examples/supervisor-loop/shared.ts index bc10d36c..e137b935 100644 --- a/examples/supervisor-loop/shared.ts +++ b/examples/supervisor-loop/shared.ts @@ -57,7 +57,10 @@ export function scriptedSupervisorChat(workerCount: number, labelPrefix = 'solve { name: 'spawn_agent', arguments: { - profile: { name: `${labelPrefix}-${i}`, systemPrompt: `Emit ${expectedAnswer}.` }, + profile: { + name: `${labelPrefix}-${i}`, + prompt: { systemPrompt: `Emit ${expectedAnswer}.` }, + }, task: `Emit the exact line ${expectedAnswer} and nothing else.`, label: `${labelPrefix}-${i}`, }, diff --git a/src/candidate-execution/profile.ts b/src/candidate-execution/profile.ts index eabec20f..93a94780 100644 --- a/src/candidate-execution/profile.ts +++ b/src/candidate-execution/profile.ts @@ -230,6 +230,7 @@ export function assertCandidateProfileExecutionSupport(profile: AgentCandidatePr } } +/** Convert the candidate profile contract into the portable interface profile it represents. */ export function agentCandidateProfileAsAgentProfile( candidate: AgentCandidateProfile, ): AgentProfile { diff --git a/src/knowledge/supervised-update.ts b/src/knowledge/supervised-update.ts index b1df286b..46f8fd2e 100644 --- a/src/knowledge/supervised-update.ts +++ b/src/knowledge/supervised-update.ts @@ -137,8 +137,8 @@ export async function runSupervisedKnowledgeUpdate( const profile: SupervisorProfile = { name: 'knowledge-research-supervisor', - model: options.supervisorModel, - systemPrompt, + ...(options.supervisorModel ? { model: { default: options.supervisorModel } } : {}), + prompt: { systemPrompt }, } const run = options.runSupervised ?? supervise const task = formatSupervisedKnowledgeTask(options) diff --git a/src/runtime/supervise/bridge-executor.test.ts b/src/runtime/supervise/bridge-executor.test.ts index 0f587ed3..f6b1b0f6 100644 --- a/src/runtime/supervise/bridge-executor.test.ts +++ b/src/runtime/supervise/bridge-executor.test.ts @@ -6,18 +6,75 @@ import { spendFromUsageEvents } from './budget' import { bridgeExecutor } from './runtime' import type { UsageEvent } from './types' +const TEST_RUN_DIGEST = `sha256:${'b'.repeat(64)}` + +function numberSseDataFrames(body: string): string { + let seq = 0 + return body.replace(/^data: (?!\[DONE\])/gmu, () => `id: ${++seq}\ndata: `) +} + +function durableRunHeaders(runId: string): Record { + return { + 'x-run-id': runId, + 'x-run-request-digest': TEST_RUN_DIGEST, + } +} + +function terminalCancelBody(runId: string): string { + return JSON.stringify({ + cancelled: true, + cancel_requested: true, + terminal: true, + run: { + id: runId, + requestDigest: TEST_RUN_DIGEST, + terminal: true, + status: 'cancelled', + state: 'terminal', + }, + }) +} + +function cancelledRunId(url: string | undefined): string | undefined { + const match = url?.match(/^\/v1\/runs\/([^/]+)\/cancel(?:\?|$)/u) + return match?.[1] ? decodeURIComponent(match[1]) : undefined +} + +function firstHeader(value: string | string[] | undefined): string | undefined { + return Array.isArray(value) ? value[0] : value +} + /** Serve one canned cli-bridge response body per request (HTTP 200 unless told * otherwise) and hand back the bridge URL — the upstream-failure shapes under * test are byte-level wire artifacts, so the test speaks real HTTP. */ async function startBridgeStub( body: string, - opts: { status?: number; contentType?: string } = {}, + opts: { + status?: number + contentType?: string + onRequest?: (body: Record) => void + } = {}, ): Promise<{ url: string; server: Server }> { - const server = createServer((_req, res) => { + const server = createServer(async (req, res) => { + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + const requestBody = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}') as Record< + string, + unknown + > + if (opts.onRequest) { + opts.onRequest(requestBody) + } res.writeHead(opts.status ?? 200, { 'content-type': opts.contentType ?? 'text/event-stream', + 'x-run-id': String(requestBody.run_id), + 'x-run-request-digest': TEST_RUN_DIGEST, }) - res.end(body) + res.end( + (opts.contentType ?? 'text/event-stream') === 'text/event-stream' + ? numberSseDataFrames(body) + : body, + ) }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const { port } = server.address() as AddressInfo @@ -127,4 +184,413 @@ describe('bridgeExecutor upstream-error propagation', () => { expect(artifact.out).toMatchObject({ content: 'first second third' }) expect(normalized).toEqual({ ...artifact.spent, ms: 0 }) }) + + it('sends the complete canonical profile once and lets it select the harness and model', async () => { + let requestBody: Record | undefined + const stub = await startBridgeStub('data: [DONE]\n\n', { + onRequest: (body) => { + requestBody = body + }, + }) + server = stub.server + const profile: AgentProfile = { + name: 'research-leader', + description: 'Design and supervise discriminating experiments', + harness: 'codex', + prompt: { + systemPrompt: 'Lead the pursuit.', + instructions: ['Prefer falsifiable hypotheses.'], + }, + model: { default: 'gpt-5.6', reasoningEffort: 'high' }, + permissions: { shell: 'ask' }, + tools: { web: true }, + mcp: { + literature: { transport: 'http', url: 'https://papers.example.test/mcp' }, + }, + subagents: { + reviewer: { description: 'Challenge the evidence', prompt: 'Find confounds.' }, + }, + resources: { + skills: [{ kind: 'inline', name: 'hypothesis', content: '# Hypothesis\nTest mechanisms.' }], + failOnError: true, + }, + hooks: { afterTool: [{ command: './record-result', blocking: true }] }, + modes: { adversarial: { prompt: 'Try to falsify the leading claim.' } }, + metadata: { role: 'driver', source: 'test-fixture' }, + } + const executor = bridgeExecutor( + { profile, harness: null }, + { + signal: new AbortController().signal, + seams: { + bridge: { bridgeUrl: stub.url, bridgeBearer: 'test-bearer', model: 'kimi-code/k2' }, + }, + }, + ) + await drain( + executor.execute( + 'design the experiment', + new AbortController().signal, + ) as AsyncIterable, + ) + + expect(requestBody?.model).toBe('codex/gpt-5.6') + expect(requestBody?.agent_profile).toEqual(profile) + expect(requestBody?.messages).toEqual([{ role: 'user', content: 'design the experiment' }]) + }) + + it('marks dollar cost unknown when the bridge reports no price', async () => { + const stub = await startBridgeStub( + `data: ${JSON.stringify({ usage: { prompt_tokens: 3, completion_tokens: 2 } })}\n\ndata: [DONE]\n\n`, + ) + server = stub.server + const executor = makeExecutor(stub.url) + await drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + expect(executor.resultArtifact().spent).toMatchObject({ + tokens: { input: 3, output: 2 }, + usd: 0, + usdKnown: false, + }) + }) + + it('keeps dollar cost unknown when a later completed turn omits price', async () => { + let requests = 0 + let deliver: (message: unknown) => void = () => {} + server = createServer(async (req, res) => { + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + const requestBody = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + > + const runId = String(requestBody.run_id) + requests += 1 + if (requests === 1) deliver({ steer: 'check the edge case too' }) + res.writeHead(200, { + 'content-type': 'text/event-stream', + ...durableRunHeaders(runId), + }) + const usage = + requests === 1 + ? { prompt_tokens: 3, completion_tokens: 2, cost: 0.01 } + : { prompt_tokens: 4, completion_tokens: 1 } + res.end( + numberSseDataFrames( + `data: ${JSON.stringify({ choices: [{ delta: { content: `turn-${requests}` } }], usage })}\n\ndata: [DONE]\n\n`, + ), + ) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + deliver = (message) => executor.deliver?.(message) + + await drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + + expect(requests).toBe(2) + expect(executor.resultArtifact().spent).toMatchObject({ + iterations: 2, + tokens: { input: 7, output: 3 }, + usd: 0.01, + usdKnown: false, + }) + }) + + it.each([ + { defect: 'changed run id', expected: /run identity mismatch/u }, + { defect: 'changed request digest', expected: /request digest changed/u }, + { defect: 'skipped replay event', expected: /replay gap: expected event 2, received 3/u }, + ])('fails closed when a reconnect has a $defect', async ({ defect, expected }) => { + const requests: Array<{ + body: Record + lastEventId: string | undefined + }> = [] + server = createServer(async (req, res) => { + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record + const runId = String(body.run_id) + requests.push({ body, lastEventId: firstHeader(req.headers['last-event-id']) }) + const responseRunId = + requests.length === 2 && defect === 'changed run id' ? `wrong-${runId}` : runId + const responseDigest = + requests.length === 2 && defect === 'changed request digest' + ? `sha256:${'c'.repeat(64)}` + : TEST_RUN_DIGEST + res.writeHead(200, { + 'content-type': 'text/event-stream', + 'x-run-id': responseRunId, + 'x-run-request-digest': responseDigest, + }) + if (requests.length === 1) { + res.end(`id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 1 } })}\n\n`) + return + } + const eventId = defect === 'skipped replay event' ? 3 : 2 + res.end( + `id: ${eventId}\ndata: ${JSON.stringify({ usage: { completion_tokens: 1 } })}\n\ndata: [DONE]\n\n`, + ) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + + await expect( + drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ), + ).rejects.toThrow(expected) + expect(requests).toHaveLength(2) + expect(requests[1]?.body).toEqual(requests[0]?.body) + expect(requests[1]?.lastEventId).toBe('1') + }) + + it('reattaches one live execution after disconnect and cancels that exact run on teardown', async () => { + const chatRequests: Array<{ + body: Record + lastEventId: string | undefined + }> = [] + const liveRuns = new Set() + const cancelledRuns: string[] = [] + let executions = 0 + let reattached: () => void = () => {} + const reattachedPromise = new Promise((resolve) => { + reattached = resolve + }) + let cancelSeen: () => void = () => {} + const cancelSeenPromise = new Promise((resolve) => { + cancelSeen = resolve + }) + let acknowledgeTerminal: () => void = () => {} + const terminalAcknowledged = new Promise((resolve) => { + acknowledgeTerminal = resolve + }) + + server = createServer(async (req, res) => { + const cancelledId = cancelledRunId(req.url) + if (cancelledId) { + cancelledRuns.push(cancelledId) + cancelSeen() + await terminalAcknowledged + liveRuns.delete(cancelledId) + res.writeHead(200, { + 'content-type': 'application/json', + ...durableRunHeaders(cancelledId), + }) + res.end(terminalCancelBody(cancelledId)) + return + } + + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record + const runId = String(body.run_id) + chatRequests.push({ body, lastEventId: firstHeader(req.headers['last-event-id']) }) + if (!liveRuns.has(runId)) { + liveRuns.add(runId) + executions += 1 + } + res.writeHead(200, { + 'content-type': 'text/event-stream', + ...durableRunHeaders(runId), + }) + if (chatRequests.length === 1) { + res.write( + `id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 5, completion_tokens: 2 } })}\n\n`, + ) + setTimeout(() => res.destroy(), 5) + return + } + reattached() + // Keep the second reader attached. Only the explicit cancel endpoint + // changes the logical run state; closing either socket does not. + res.write(': attached\n\n') + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + const iterator = ( + executor.execute('do the task', new AbortController().signal) as AsyncIterable + )[Symbol.asyncIterator]() + + expect(await iterator.next()).toMatchObject({ + value: { kind: 'tokens', input: 5, output: 2 }, + done: false, + }) + const draining = drain({ [Symbol.asyncIterator]: () => iterator }).catch((error) => error) + await reattachedPromise + + expect(executions).toBe(1) + expect(liveRuns.size).toBe(1) + expect(chatRequests).toHaveLength(2) + expect(chatRequests[1]?.body).toEqual(chatRequests[0]?.body) + expect(chatRequests[1]?.lastEventId).toBe('1') + + let teardownSettled = false + const teardown = executor.teardown('infinity').then((receipt) => { + teardownSettled = true + return receipt + }) + await cancelSeenPromise + await Promise.resolve() + expect(teardownSettled).toBe(false) + expect(liveRuns.size).toBe(1) + acknowledgeTerminal() + await expect(teardown).resolves.toEqual({ destroyed: true }) + await draining + expect(cancelledRuns).toEqual([chatRequests[0]?.body.run_id]) + expect(liveRuns.size).toBe(0) + }) + + it('interrupts an active response body, accounts its partial usage, and resumes with the steer', async () => { + const requestBodies: Array> = [] + const liveRuns = new Set() + const cancelledRuns: string[] = [] + server = createServer(async (req, res) => { + const cancelledId = cancelledRunId(req.url) + if (cancelledId) { + cancelledRuns.push(cancelledId) + liveRuns.delete(cancelledId) + res.writeHead(200, { + 'content-type': 'application/json', + ...durableRunHeaders(cancelledId), + }) + res.end(terminalCancelBody(cancelledId)) + return + } + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + const requestBody = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + > + requestBodies.push(requestBody) + const runId = String(requestBody.run_id) + liveRuns.add(runId) + res.writeHead(200, { + 'content-type': 'text/event-stream', + ...durableRunHeaders(runId), + }) + if (requestBodies.length === 1) { + res.write( + `id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 5, completion_tokens: 2 } })}\n\n`, + ) + return + } + res.end( + numberSseDataFrames( + `data: ${JSON.stringify({ + choices: [{ delta: { content: 'corrected answer' } }], + usage: { prompt_tokens: 3, completion_tokens: 1, cost: 0.01 }, + })}\n\ndata: [DONE]\n\n`, + ), + ) + liveRuns.delete(runId) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + const iterator = ( + executor.execute('do the task', new AbortController().signal) as AsyncIterable + )[Symbol.asyncIterator]() + + expect(await iterator.next()).toMatchObject({ + value: { kind: 'tokens', input: 5, output: 2 }, + done: false, + }) + executor.deliver?.({ steer: 'stop and use the corrected method', interrupt: true }) + const remaining = await drain({ [Symbol.asyncIterator]: () => iterator }) + + expect(remaining).toContainEqual({ kind: 'tokens', input: 3, output: 1 }) + expect(requestBodies).toHaveLength(2) + expect(cancelledRuns).toEqual([requestBodies[0]?.run_id]) + expect(liveRuns.size).toBe(0) + expect(requestBodies[1]?.messages).toEqual([ + { + role: 'user', + content: expect.stringContaining('stop and use the corrected method'), + }, + ]) + expect(executor.resultArtifact()).toMatchObject({ + out: { content: 'corrected answer' }, + spent: { + iterations: 2, + tokens: { input: 8, output: 3 }, + usd: 0.01, + usdKnown: false, + }, + }) + }) + + it('marks pre-header interrupted bridge work unknown instead of treating it as free', async () => { + const requestBodies: Array> = [] + const cancelledRuns: string[] = [] + let firstRequestSeen: () => void = () => {} + const firstRequest = new Promise((resolve) => { + firstRequestSeen = resolve + }) + server = createServer(async (req, res) => { + const cancelledId = cancelledRunId(req.url) + if (cancelledId) { + cancelledRuns.push(cancelledId) + res.writeHead(200, { + 'content-type': 'application/json', + ...durableRunHeaders(cancelledId), + }) + res.end(terminalCancelBody(cancelledId)) + return + } + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + const requestBody = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + > + requestBodies.push(requestBody) + if (requestBodies.length === 1) { + firstRequestSeen() + return + } + const runId = String(requestBody.run_id) + res.writeHead(200, { + 'content-type': 'text/event-stream', + ...durableRunHeaders(runId), + }) + res.end( + numberSseDataFrames( + `data: ${JSON.stringify({ + choices: [{ delta: { content: 'resumed answer' } }], + usage: { prompt_tokens: 3, completion_tokens: 1, cost: 0.01 }, + })}\n\ndata: [DONE]\n\n`, + ), + ) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + const draining = drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + + await firstRequest + executor.deliver?.({ steer: 'resume with the corrected plan', interrupt: true }) + await draining + + expect(requestBodies).toHaveLength(2) + expect(cancelledRuns).toEqual([requestBodies[0]?.run_id]) + expect(executor.resultArtifact()).toMatchObject({ + out: { content: 'resumed answer' }, + spent: { + iterations: 2, + tokens: { input: 3, output: 1 }, + tokensKnown: false, + usd: 0.01, + usdKnown: false, + }, + }) + }) }) diff --git a/src/runtime/supervise/delegate.ts b/src/runtime/supervise/delegate.ts index f03a17d4..e9932149 100644 --- a/src/runtime/supervise/delegate.ts +++ b/src/runtime/supervise/delegate.ts @@ -58,24 +58,27 @@ export interface DelegateOptions { readonly brain?: ToolLoopChat /** Override the default authoring-supervisor profile (name / extra system-prompt stance). The * default already carries the authoring skill; override only to add a goal or rename. */ - readonly supervisor?: Partial> + readonly supervisor?: { + readonly name?: string + readonly systemPrompt?: string + } /** Restrict the run to this subset of models (forwarded to `supervise()`). */ readonly allowedModels?: readonly string[] readonly runId?: string } -/** Build the DEFAULT authoring supervisor profile: a router-brained supervisor (`harness: null`) +/** Build the DEFAULT authoring supervisor profile: a router-brained supervisor (`harness: cli-base`) * whose standing instruction IS the authoring-agent-profiles skill, so it decomposes the intent and * AUTHORS a worker profile per sub-task. No worker profile is baked in here. */ function authoringSupervisorProfile( model: string | undefined, - override?: Partial>, + override?: { readonly name?: string; readonly systemPrompt?: string }, ): SupervisorProfile { return { name: override?.name ?? 'delegate-supervisor', - harness: null, - ...(model ? { model } : {}), - systemPrompt: override?.systemPrompt ?? supervisorInstructions(), + harness: 'cli-base', + ...(model ? { model: { default: model } } : {}), + prompt: { systemPrompt: override?.systemPrompt ?? supervisorInstructions() }, } } diff --git a/src/runtime/supervise/dispatch.ts b/src/runtime/supervise/dispatch.ts index 3e98d1c4..6a1222d5 100644 --- a/src/runtime/supervise/dispatch.ts +++ b/src/runtime/supervise/dispatch.ts @@ -20,9 +20,11 @@ * Three unrelated caps bound "how much runs at once" in this stack, at three different layers. * They are NOT aware of each other, and the smallest one silently wins: * - * 1. `CoordinationToolsOptions.maxLiveWorkers` (`src/mcp/tools/coordination.ts`) — supervisor - * level. How many workers may be spawned-but-not-settled at once; `spawn_agent` fails closed - * with `error: 'max-live-workers'` past it. Unset by default ⇒ NO cap at this layer. + * 1. `SuperviseOptions.maxLiveWorkers` (`src/runtime/supervise/supervise.ts`) — supervised-tree + * level. One shared Scope counter bounds every spawned manager and leaf in the recursive tree; + * `spawn_agent` fails closed with `error: 'max-live-workers'` past it. Unset ⇒ NO tree cap. + * `CoordinationToolsOptions.maxLiveWorkers` remains the local form for a toolbox mounted on a + * caller-owned Scope that has no tree limit. * 2. `SandboxLineage`'s `maxConcurrency` / `DEFAULT_FORK_CONCURRENCY = 4` * (`src/runtime/sandbox-lineage.ts`) — kernel level. How many BOXES one `runAgentRounds` fork wave * provisions at once. It bounds a single leaf's fanout, not the supervisor's worker count. diff --git a/src/runtime/supervise/model-policy.ts b/src/runtime/supervise/model-policy.ts index 150d4b4e..4bb705bb 100644 --- a/src/runtime/supervise/model-policy.ts +++ b/src/runtime/supervise/model-policy.ts @@ -4,6 +4,7 @@ * model at resolve time, so a run that names a model outside the allowed set throws before * any compute is spent — never silently swapped or silently allowed. */ +import type { AgentProfile } from '@tangle-network/agent-interface' import { ConfigError } from '../../errors' /** @@ -22,3 +23,19 @@ export function assertModelAllowed( ) } } + +/** Check every canonical model-bearing field in a complete profile, including the models a + * backend may select for cheap work, named subagents, or modes. */ +export function assertProfileModelsAllowed( + profile: AgentProfile, + allowed: readonly string[] | undefined, +): void { + assertModelAllowed(profile.model?.default, allowed) + assertModelAllowed(profile.model?.small, allowed) + for (const subagent of Object.values(profile.subagents ?? {})) { + assertModelAllowed(subagent.model, allowed) + } + for (const mode of Object.values(profile.modes ?? {})) { + assertModelAllowed(mode.model, allowed) + } +} diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index fde6ad75..a46bd6e8 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -443,6 +443,9 @@ function isAsyncIterable(value: unknown): value is AsyncIterable { export interface SuperviseOptions { /** The conserved compute pool for the whole run. */ readonly budget: Budget + /** Caller-owned cancellation for the complete recursive run. Aborting it cascades through the + * root scope and every live child, including acquisition and backend execution. */ + readonly signal?: AbortSignal /** Trusted candidate and pursuit attribution for the root. The runtime derives profile/task * digests itself from the exact detached values it executes. */ readonly execution?: AgentExecutionRef @@ -667,6 +670,7 @@ function captureSuperviseOptions(opts: SuperviseOptions): SuperviseOptions { onProgressStop, finalizer, now, + signal, ...decisionData } = opts const capturedData = detachedSnapshot(decisionData, 'supervise options') @@ -750,6 +754,7 @@ function captureSuperviseOptions(opts: SuperviseOptions): SuperviseOptions { ...(onProgressStop === undefined ? {} : { onProgressStop }), ...(finalizer === undefined ? {} : { finalizer }), ...(now === undefined ? {} : { now }), + ...(signal === undefined ? {} : { signal }), }) } @@ -1235,6 +1240,7 @@ export function supervise(profile: SupervisorProfile, task: unknown, opts: Super ...(options.probes ? { probes: options.probes } : {}), ...(ctx.resume === true ? { resume: true } : {}), ...(options.now ? { now: options.now } : {}), + ...(options.signal ? { signal: options.signal } : {}), }) } diff --git a/tests/helpers/resume-driver-child.ts b/tests/helpers/resume-driver-child.ts index a986c8f7..f2c1759a 100644 --- a/tests/helpers/resume-driver-child.ts +++ b/tests/helpers/resume-driver-child.ts @@ -139,7 +139,7 @@ const brain: ToolLoopChat = async (messages) => { } } -const result = await supervise({ name: 'root', harness: null }, 'five assignments', { +const result = await supervise({ name: 'root', harness: 'cli-base' }, 'five assignments', { budget: { maxIterations: 200, maxTokens: 500_000 }, // Explicit per-worker ceiling: the default is a quarter of the pool, which would starve the // fifth spawn and make this a four-worker test. diff --git a/tests/kernel/completion-gate.test.ts b/tests/kernel/completion-gate.test.ts index b5691bce..650130b8 100644 --- a/tests/kernel/completion-gate.test.ts +++ b/tests/kernel/completion-gate.test.ts @@ -127,7 +127,7 @@ let blobs = new InMemoryResultBlobStore() function driverOpts( name: string, brain: ToolLoopChat, - makeWorkerAgent: (p: unknown) => Agent, + makeWorkerAgent: (p: AgentProfile) => Agent, ): DriverAgentOptions { return { name, brain, blobs, makeWorkerAgent, perWorker, systemPrompt: 'drive', maxTurns: 8 } } @@ -149,7 +149,14 @@ function gatedWorkerLeaf( } const spawnAwaitStop: ScriptedTurn[] = [ - { toolCalls: [{ name: 'spawn_agent', arguments: { profile: { kind: 'worker' }, task: 'go' } }] }, + { + toolCalls: [ + { + name: 'spawn_agent', + arguments: { profile: { metadata: { kind: 'worker' } }, task: 'go' }, + }, + ], + }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, { content: 'stop' }, ] @@ -207,14 +214,19 @@ describe('completion-oracle settle — settled ⟺ DELIVERED (Foreman 0/18)', () { out: { pick: 'not-me' }, score: 0.99 }, { check: () => false }, ) - const makeAgent = (raw: unknown) => - (raw as { which?: string })?.which === 'b' ? ran : delivered + const makeAgent = (profile: AgentProfile) => (profile.metadata?.which === 'b' ? ran : delivered) // spawn BOTH, await BOTH, stop. const turns: ScriptedTurn[] = [ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { which: 'a' }, task: 'a' } }, - { name: 'spawn_agent', arguments: { profile: { which: 'b' }, task: 'b' } }, + { + name: 'spawn_agent', + arguments: { profile: { metadata: { which: 'a' } }, task: 'a' }, + }, + { + name: 'spawn_agent', + arguments: { profile: { metadata: { which: 'b' } }, task: 'b' }, + }, ], }, { @@ -244,9 +256,8 @@ describe('completion-oracle settle — settled ⟺ DELIVERED (Foreman 0/18)', () const journal = new InMemorySpawnJournal() // The mid driver spawns ONE worker whose deliverable check FAILS. - const makeAgent = (raw: unknown): Agent => { - const p = raw as { kind?: string } - if (p?.kind === 'driver') { + const makeAgent = (profile: AgentProfile): Agent => { + if (profile.metadata?.kind === 'driver') { return driverChild( 'mid', driverAgent(driverOpts('mid', scriptedBrain(spawnAwaitStop), makeAgent)), @@ -262,7 +273,10 @@ describe('completion-oracle settle — settled ⟺ DELIVERED (Foreman 0/18)', () const rootTurns: ScriptedTurn[] = [ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'driver' }, task: 'delegate' } }, + { + name: 'spawn_agent', + arguments: { profile: { metadata: { kind: 'driver' } }, task: 'delegate' }, + }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, diff --git a/tests/kernel/coordination-driver.test.ts b/tests/kernel/coordination-driver.test.ts index 2ef776e1..1559f3f2 100644 --- a/tests/kernel/coordination-driver.test.ts +++ b/tests/kernel/coordination-driver.test.ts @@ -31,7 +31,7 @@ interface WorkerScript { readonly score: number } -function workerExecutor(s: WorkerScript): Executor { +function workerExecutor(s: WorkerScript, onTeardown?: () => void): Executor { const events: UsageEvent[] = [] for (let i = 0; i < s.iterations; i += 1) events.push({ kind: 'iteration' }) events.push({ kind: 'tokens', input: s.tokens.input, output: s.tokens.output }) @@ -42,7 +42,10 @@ function workerExecutor(s: WorkerScript): Executor { for (const ev of events) yield ev })() }, - teardown: () => Promise.resolve({ destroyed: true }), + teardown: () => { + onTeardown?.() + return Promise.resolve({ destroyed: true }) + }, resultArtifact(): ExecutorResult { return { outRef: `w:${JSON.stringify(s.out)}`, @@ -54,11 +57,15 @@ function workerExecutor(s: WorkerScript): Executor { } } -function workerLeaf(name: string, s: WorkerScript): Agent { +function workerLeaf( + name: string, + s: WorkerScript, + onTeardown?: () => void, +): Agent { const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, - executor: workerExecutor(s), + executor: workerExecutor(s, onTeardown), } return { name, act: async () => s.out, executorSpec: spec } as Agent & { executorSpec: AgentSpec @@ -101,7 +108,7 @@ const perWorker: Budget = { maxIterations: 4, maxTokens: 1000 } function driverOpts( name: string, brain: ToolLoopChat, - makeWorkerAgent: (p: unknown) => Agent, + makeWorkerAgent: (p: AgentProfile) => Agent, ): DriverAgentOptions { return { name, @@ -130,14 +137,17 @@ describe('driverAgent — the driver BRAIN (LLM tool-loop drives real spawns)', score: 0.9, }) // The makeWorkerAgent the spawn_agent tool dispatches: this test only spawns the worker leaf. - const makeAgent = (_p: unknown): Agent => worker + const makeAgent = (_p: AgentProfile): Agent => worker // Scripted driver LLM: turn 0 spawns a worker, turn 1 awaits it, turn 2 stops (no calls). const chat = scriptedBrain( [ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'worker' }, task: 'go' } }, + { + name: 'spawn_agent', + arguments: { profile: { metadata: { kind: 'worker' } }, task: 'go' }, + }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, @@ -180,25 +190,45 @@ describe('driverAgent — the driver BRAIN (LLM tool-loop drives real spawns)', SHARED_BLOBS = new InMemoryResultBlobStore() const journal = new InMemorySpawnJournal() - const worker = workerLeaf('w', { - out: { answer: 42 }, - tokens: { input: 10, output: 5 }, - iterations: 1, - score: 0.9, + let markWorkerFinished: (() => void) | undefined + const workerFinished = new Promise((resolve) => { + markWorkerFinished = resolve }) - const makeAgent = (_p: unknown): Agent => worker + + const worker = workerLeaf( + 'w', + { + out: { answer: 42 }, + tokens: { input: 10, output: 5 }, + iterations: 1, + score: 0.9, + }, + () => markWorkerFinished?.(), + ) + const makeAgent = (_p: AgentProfile): Agent => worker // Scripted driver LLM: spawns a worker then STOPS — it never calls await_event, the exact // pull-discipline failure a live LLM brain exhibits. The worker still delivers; losing it // to an empty ledger was the bug. - const chat = scriptedBrain([ + const scripted = scriptedBrain([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'worker' }, task: 'go' } }, + { + name: 'spawn_agent', + arguments: { profile: { metadata: { kind: 'worker' } }, task: 'go' }, + }, ], }, { content: 'spawned; stopping without awaiting' }, ]) + let turn = 0 + const chat: ToolLoopChat = async (messages, options) => { + // Model inference naturally leaves time between tool rounds. Make that ordering explicit so + // this test proves the documented case—already-settled work—not a race with journal commit. + if (turn === 1) await workerFinished + turn += 1 + return scripted(messages, options) + } const root = driverAgent(driverOpts('root', chat, makeAgent)) const result = await createSupervisor().run(root, 'solve it', { @@ -228,7 +258,7 @@ describe('driverAgent — the driver BRAIN (LLM tool-loop drives real spawns)', // Alternate good/failing on each spawn_agent dispatch — the brain fans out three workers, // two of which crash (down), and stops without awaiting any of them. let spawn = 0 - const makeAgent = (_p: unknown): Agent => + const makeAgent = (_p: AgentProfile): Agent => spawn++ === 0 ? good : hangingWorkerLeaf(`bad-${spawn}`) const chat = scriptedBrain([ @@ -279,7 +309,10 @@ describe('driverAgent — the driver BRAIN (LLM tool-loop drives real spawns)', const midTurns: ScriptedTurn[] = [ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'worker' }, task: 'sub' } }, + { + name: 'spawn_agent', + arguments: { profile: { metadata: { kind: 'worker' } }, task: 'sub' }, + }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, @@ -288,9 +321,8 @@ describe('driverAgent — the driver BRAIN (LLM tool-loop drives real spawns)', // The recursive resolver: a 'driver' profile → a driverChild wrapping ANOTHER // driverAgent (over the same recursive makeAgent); a 'worker' profile → leaf. - const makeAgent = (raw: unknown): Agent => { - const p = raw as { kind?: string } - if (p?.kind === 'driver') { + const makeAgent = (profile: AgentProfile): Agent => { + if (profile.metadata?.kind === 'driver') { const childBrain = scriptedBrain(midTurns, midSeen) return driverChild('mid', driverAgent(driverOpts('mid', childBrain, makeAgent)), journal) } @@ -302,7 +334,10 @@ describe('driverAgent — the driver BRAIN (LLM tool-loop drives real spawns)', [ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'driver' }, task: 'delegate' } }, + { + name: 'spawn_agent', + arguments: { profile: { metadata: { kind: 'driver' } }, task: 'delegate' }, + }, ], }, { toolCalls: [{ name: 'await_event', arguments: {} }] }, @@ -424,7 +459,7 @@ function collectTreeKeys(journal: InMemorySpawnJournal): string[] { // `list_questions` is always present (no analysts needed), has no side effects, and reserves no // budget — the ideal benign tool for driving the loop a fixed number of turns. const benignTurn: ScriptedTurn = { toolCalls: [{ name: 'list_questions', arguments: {} }] } -const dummyWorker = (_p: unknown): Agent => +const dummyWorker = (_p: AgentProfile): Agent => workerLeaf('w', { out: {}, tokens: { input: 0, output: 0 }, iterations: 0, score: 0 }) function bounds0Opts(name: string, brain: ToolLoopChat): DriverAgentOptions { @@ -645,7 +680,7 @@ describe('driverAgent — the driver can ACT (call work tools itself), not only }) describe('driverAgent — the analyst up-leg (analysts + analyzeOnSettle pass-through)', () => { - const noWorker = (_p: unknown): Agent => + const noWorker = (_p: AgentProfile): Agent => ({ name: 'w', act: async () => '', diff --git a/tests/kernel/coordination-log.test.ts b/tests/kernel/coordination-log.test.ts new file mode 100644 index 00000000..d7950740 --- /dev/null +++ b/tests/kernel/coordination-log.test.ts @@ -0,0 +1,265 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { InMemoryResultBlobStore } from '../../src/durable/spawn-journal' +import type { ContinuationInstruction, CoordinationEvent } from '../../src/mcp/tools/coordination' +import { createCoordinationTools } from '../../src/mcp/tools/coordination' +import { FileCoordinationLog } from '../../src/runtime/supervise/coordination-log' +import type { BusRecord } from '../../src/runtime/supervise/event-bus' +import type { Agent, Scope } from '../../src/runtime/supervise/types' + +function stamped(seq: number, event: CoordinationEvent): BusRecord { + return { seq, at: 1_000 + seq, priority: event.type === 'question' ? 20 : 0, event } +} + +function instruction(receiptId: string, text: string): ContinuationInstruction { + return { + receiptId, + kind: 'steer', + toWorker: 'worker-1', + instruction: text, + instructionDigest: `sha256:${receiptId.padEnd(64, '0').slice(0, 64)}`, + interrupt: false, + } +} + +function evidenceChain( + receipt: ContinuationInstruction, + outcome: 'delivered' | 'runtime-has-no-inbox', +): CoordinationEvent[] { + return [ + { type: 'instruction', instruction: receipt }, + { + type: 'delivery-attempt', + attempt: { + receiptId: receipt.receiptId, + kind: receipt.kind, + toWorker: receipt.toWorker, + instructionDigest: receipt.instructionDigest, + interrupt: receipt.interrupt, + }, + }, + { + type: 'steer', + down: { + receiptId: receipt.receiptId, + toWorker: receipt.toWorker, + instruction: receipt.instruction, + instructionDigest: receipt.instructionDigest, + delivered: outcome === 'delivered', + outcome, + }, + }, + ] +} + +describe('FileCoordinationLog delivery evidence', () => { + it('keeps complete success and refusal chains linked and isolated by stable owner', async () => { + const dir = await mkdtemp(join(tmpdir(), 'coordination-log-owner-')) + try { + const log = new FileCoordinationLog(join(dir, 'coordination.jsonl')) + const accepted = instruction('accepted', 'continue A') + const refused = instruction('refused', 'continue B') + for (const [seq, event] of evidenceChain(accepted, 'delivered').entries()) { + await log.append('run', stamped(seq, event), 'owner-A') + } + for (const [seq, event] of evidenceChain(refused, 'runtime-has-no-inbox').entries()) { + await log.append('run', stamped(seq, event), 'owner-B') + } + + const ownerA = await log.load('run', 'owner-A') + const ownerB = await log.load('run', 'owner-B') + expect(ownerA.continuations.map((entry) => entry.receiptId)).toEqual(['accepted']) + expect(ownerA.deliveryEvidence.map((entry) => entry.type)).toEqual([ + 'delivery-attempt', + 'steer', + ]) + expect(ownerB.continuations.map((entry) => entry.receiptId)).toEqual(['refused']) + expect(ownerB.deliveryEvidence.map((entry) => entry.type)).toEqual([ + 'delivery-attempt', + 'steer', + ]) + const acceptedOutcome = ownerA.deliveryEvidence.find((entry) => entry.type === 'steer') + const refusedOutcome = ownerB.deliveryEvidence.find((entry) => entry.type === 'steer') + expect(acceptedOutcome?.down).toMatchObject({ receiptId: 'accepted', outcome: 'delivered' }) + expect(refusedOutcome?.down).toMatchObject({ + receiptId: 'refused', + outcome: 'runtime-has-no-inbox', + }) + expect(ownerA.records.map(({ seq, at, priority }) => ({ seq, at, priority }))).toEqual([ + { seq: 0, at: 1000, priority: 0 }, + { seq: 1, at: 1001, priority: 0 }, + { seq: 2, at: 1002, priority: 0 }, + ]) + + const raw = (await readFile(join(dir, 'coordination.jsonl'), 'utf8')) + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { ownerId?: string }) + expect(raw.map((record) => record.ownerId)).toEqual([ + 'owner-A', + 'owner-A', + 'owner-A', + 'owner-B', + 'owner-B', + 'owner-B', + ]) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + it('preserves an attempted-but-unconfirmed delivery as unknown evidence without inventing an outcome', async () => { + const dir = await mkdtemp(join(tmpdir(), 'coordination-log-crash-window-')) + try { + const log = new FileCoordinationLog(join(dir, 'coordination.jsonl')) + const receipt = instruction('crashed', 'do not replay me') + const [authorized, attempted] = evidenceChain(receipt, 'delivered') + if (!authorized || !attempted) throw new Error('missing evidence fixture') + await log.append('run', stamped(0, authorized), 'owner') + await log.append('run', stamped(1, attempted), 'owner') + + const prior = await log.load('run', 'owner') + expect(prior.continuations.map((entry) => entry.receiptId)).toEqual(['crashed']) + expect(prior.deliveryEvidence).toEqual([attempted]) + expect( + prior.deliveryEvidence.some((entry) => entry.type === 'steer' || entry.type === 'answer'), + ).toBe(false) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + it('keeps a question blocking after a refused answer delivery across restart', async () => { + const dir = await mkdtemp(join(tmpdir(), 'coordination-log-refused-answer-')) + try { + const log = new FileCoordinationLog(join(dir, 'coordination.jsonl')) + const questionId = 'worker-1:q0' + const receipt: ContinuationInstruction = { + ...instruction('answer-refused', 'choose B'), + kind: 'answer', + questionId, + } + const question: CoordinationEvent = { + type: 'question', + question: { + id: questionId, + from: 'worker-1', + level: 'worker', + question: 'Which target?', + reason: 'the experiment cannot continue without a target', + urgency: 'blocks-run', + status: 'open', + openedAt: 0, + }, + } + const attempt: CoordinationEvent = { + type: 'delivery-attempt', + attempt: { + receiptId: receipt.receiptId, + kind: 'answer', + toWorker: receipt.toWorker, + instructionDigest: receipt.instructionDigest, + interrupt: false, + questionId, + }, + } + const refused: CoordinationEvent = { + type: 'answer', + questionId, + down: { + receiptId: receipt.receiptId, + toWorker: receipt.toWorker, + instruction: receipt.instruction, + instructionDigest: receipt.instructionDigest, + delivered: false, + outcome: 'runtime-has-no-inbox', + }, + } + for (const [seq, event] of [ + question, + { type: 'instruction', instruction: receipt }, + attempt, + refused, + ].entries() as ArrayIterator<[number, CoordinationEvent]>) { + await log.append('run', stamped(seq, event), 'owner') + } + + const prior = await log.load('run', 'owner') + expect(prior.questions).toMatchObject([{ id: questionId, status: 'open' }]) + expect(prior.deliveryEvidence.at(-1)).toMatchObject({ + type: 'answer', + down: { receiptId: receipt.receiptId, delivered: false }, + }) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + it('serializes concurrent appends and replays the exact bus ordering metadata', async () => { + const dir = await mkdtemp(join(tmpdir(), 'coordination-log-concurrent-order-')) + try { + const log = new FileCoordinationLog(join(dir, 'coordination.jsonl')) + const scope = { + send: () => false, + next: async () => null, + get view() { + return { root: 'run', nodes: [], inFlight: 0, waiting: 0 } + }, + budget: { + tokensLeft: 100, + tokensKnown: true, + usdLeft: 0, + usdCapped: false, + usdKnown: true, + iterationsLeft: 10, + deadlineMs: 0, + reservedTokens: 0, + }, + signal: new AbortController().signal, + } as unknown as Scope + const coord = createCoordinationTools({ + scope, + blobs: new InMemoryResultBlobStore(), + makeWorkerAgent: () => + ({ name: 'unused', act: async () => undefined }) as Agent, + perWorker: { maxIterations: 1, maxTokens: 10 }, + onEvent: (_event, record) => log.append('run', record, 'owner'), + }) + const steer = coord.tools.find((tool) => tool.name === 'steer_agent') + if (!steer) throw new Error('steer_agent tool missing') + + await Promise.all([ + steer.handler({ workerId: 'missing-A', instruction: 'continue A' }), + steer.handler({ workerId: 'missing-B', instruction: 'continue B' }), + ]) + + const prior = await log.load('run', 'owner') + expect(prior.records.map((record) => record.seq)).toEqual([0, 1, 2, 3, 4, 5]) + for (const receipt of prior.continuations) { + const receiptSeq = prior.records.find( + (record) => + record.event.type === 'instruction' && + record.event.instruction.receiptId === receipt.receiptId, + )?.seq + const attemptSeq = prior.records.find( + (record) => + record.event.type === 'delivery-attempt' && + record.event.attempt.receiptId === receipt.receiptId, + )?.seq + const outcomeSeq = prior.records.find( + (record) => + record.event.type === 'steer' && record.event.down.receiptId === receipt.receiptId, + )?.seq + expect(receiptSeq).toBeTypeOf('number') + expect(attemptSeq).toBeGreaterThan(receiptSeq as number) + expect(outcomeSeq).toBeGreaterThan(attemptSeq as number) + } + expect(prior.records.every((record) => record.priority === 0)).toBe(true) + expect(prior.records.every((record) => Number.isFinite(record.at))).toBe(true) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/kernel/delegate.test.ts b/tests/kernel/delegate.test.ts index 715d776c..f46f4461 100644 --- a/tests/kernel/delegate.test.ts +++ b/tests/kernel/delegate.test.ts @@ -62,14 +62,14 @@ describe('delegate — the one generic delegation verb over supervise()', () => expect(superviseSpy).toHaveBeenCalledTimes(1) const [profile, task, opts] = superviseSpy.mock.calls[0] as [ - { name?: string; harness?: unknown; systemPrompt?: string }, + { name?: string; harness?: unknown; prompt?: { systemPrompt?: string } }, unknown, { backend?: unknown; router?: unknown; budget?: unknown }, ] // A router-brained AUTHORING supervisor: its standing instruction IS the authoring skill, so it // writes its own worker profile from the intent — no worker profile is baked into delegate. - expect(profile.harness ?? null).toBeNull() - expect(profile.systemPrompt).toBe(supervisorInstructions()) + expect(profile.harness).toBe('cli-base') + expect(profile.prompt?.systemPrompt).toBe(supervisorInstructions()) // The intent is handed through verbatim as the task. expect(task).toBe('fix the failing auth test') // The injected substrate (where workers run + the brain) is forwarded. @@ -127,11 +127,11 @@ describe('delegate — the one generic delegation verb over supervise()', () => }) const [profile, , opts] = superviseSpy.mock.calls[0] as [ - { model?: string }, + { model?: { default?: string } }, unknown, Record, ] - expect(profile.model).toBe('glm-5.2') + expect(profile.model?.default).toBe('glm-5.2') expect(opts.deliverable).toBe(deliverable) expect(opts.budget).toBe(budget) expect(opts.allowedModels).toEqual(['glm-5.2', 'deepseek-v4-flash']) @@ -144,9 +144,11 @@ describe('delegate — the one generic delegation verb over supervise()', () => router, supervisor: { name: 'my-supervisor', systemPrompt: 'custom stance' }, }) - const [profile] = superviseSpy.mock.calls[0] as [{ name?: string; systemPrompt?: string }] + const [profile] = superviseSpy.mock.calls[0] as [ + { name?: string; prompt?: { systemPrompt?: string } }, + ] expect(profile.name).toBe('my-supervisor') - expect(profile.systemPrompt).toBe('custom stance') + expect(profile.prompt?.systemPrompt).toBe('custom stance') }) it('fails loud on an empty intent', async () => { diff --git a/tests/kernel/driver-inference-metering.test.ts b/tests/kernel/driver-inference-metering.test.ts index c48106ac..7ad3223c 100644 --- a/tests/kernel/driver-inference-metering.test.ts +++ b/tests/kernel/driver-inference-metering.test.ts @@ -69,6 +69,55 @@ function meteredChat(turns: ScriptedTurn[]): ToolLoopChat { const perWorker: Budget = { maxIterations: 4, maxTokens: 1000 } describe("driver inference metering — the driver's own tokens count against the conserved pool", () => { + it('charges a nested worker once and releases the manager reservation', async () => { + const blobs = new InMemoryResultBlobStore() + const journal = new InMemorySpawnJournal() + const worker = workerLeaf('leaf', { input: 10, output: 0 }) + const childBudget: Budget = { maxIterations: 4, maxTokens: 40 } + + const nestedDriver: Agent = { + name: 'nested', + async act(_task, scope) { + const spawned = scope.spawn(worker, 'work', { budget: childBudget, label: 'leaf' }) + if (!spawned.ok) throw new Error(`nested spawn failed: ${spawned.reason}`) + const settled = await scope.next() + return settled?.kind === 'done' ? settled.out : undefined + }, + } + const nested = driverChild( + { name: 'nested', metadata: { role: 'driver' } }, + nestedDriver, + journal, + ) + const root: Agent = { + name: 'root', + async act(_task, scope) { + const spawned = scope.spawn(nested, 'nested work', { + budget: childBudget, + label: 'nested', + }) + if (!spawned.ok) throw new Error(`root spawn failed: ${spawned.reason}`) + await scope.next() + return scope.budget.tokensLeft + }, + } + + const result = await createSupervisor().run(root, 'task', { + budget: { maxIterations: 10, maxTokens: 100 }, + runId: 'nested-budget-once', + journal, + blobs, + executors: withDriverExecutor(createExecutorRegistry()), + maxDepth: 4, + now: () => 0, + }) + + expect(result.kind).toBe('winner') + if (result.kind !== 'winner') return + expect(result.out).toBe(90) + expect(result.spentTotal.tokens).toEqual({ input: 10, output: 0 }) + }) + it('folds driver inference into spentTotal and exposes the driver-vs-child breakdown', async () => { const blobs = new InMemoryResultBlobStore() const journal = new InMemorySpawnJournal() @@ -127,43 +176,59 @@ describe("driver inference metering — the driver's own tokens count against th const blobs = new InMemoryResultBlobStore() const journal = new InMemorySpawnJournal() const worker = workerLeaf('leaf', { input: 10, output: 5 }) + const nestedPerWorker: Budget = { ...perWorker, maxUsd: 1 } // root driver → mid sub-driver → worker leaf. The recursive resolver: a 'driver' profile becomes // a driverChild wrapping another driverAgent; a 'worker' profile becomes the leaf. - type P = { kind: 'driver'; name: string; turns: ScriptedTurn[] } | { kind: 'worker' } - const driverOf = (name: string, brain: ToolLoopChat): DriverAgentOptions => ({ + const driverOf = ( + name: string, + brain: ToolLoopChat, + workerBudget: Budget = nestedPerWorker, + ): DriverAgentOptions => ({ name, brain, blobs, makeWorkerAgent: makeAgent, - perWorker, + perWorker: workerBudget, systemPrompt: 'drive', maxTurns: 8, }) - function makeAgent(raw: unknown): Agent { - const p = raw as P - if (p?.kind === 'driver') { - return driverChild(p.name, driverAgent(driverOf(p.name, meteredChat(p.turns))), journal) - } - return worker - } // mid sub-driver inference = 60/40 + 30/20 + 10/5 = 100/65 tokens, $0.05 (re-homed up). - const midProfile: P = { - kind: 'driver', - name: 'mid', - turns: [ - { - toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'worker' }, task: 'sub' } }, - ], - usage: { input: 60, output: 40 }, - costUsd: 0.05, - }, - { toolCalls: [{ name: 'await_event', arguments: {} }], usage: { input: 30, output: 20 } }, - { content: 'mid done', usage: { input: 10, output: 5 } }, - ], + const midTurns: ScriptedTurn[] = [ + { + toolCalls: [ + { + name: 'spawn_agent', + arguments: { profile: { metadata: { kind: 'worker' } }, task: 'sub' }, + }, + ], + usage: { input: 60, output: 40 }, + costUsd: 0.05, + }, + { toolCalls: [{ name: 'await_event', arguments: {} }], usage: { input: 30, output: 20 } }, + { content: 'mid done', usage: { input: 10, output: 5 } }, + ] + function makeAgent( + profile: AgentProfile, + context?: { readonly budget: Budget }, + ): Agent { + if (profile.metadata?.kind === 'driver') { + if (!context) throw new Error('driver spawn context missing') + const childBudget: Budget = { + maxIterations: context.budget.maxIterations, + maxTokens: Math.max(1, Math.floor(context.budget.maxTokens / 4)), + ...(context.budget.maxUsd !== undefined ? { maxUsd: context.budget.maxUsd / 4 } : {}), + } + return driverChild( + 'mid', + driverAgent(driverOf('mid', meteredChat(midTurns), childBudget)), + journal, + ) + } + return worker } + const midProfile: AgentProfile = { name: 'mid', metadata: { kind: 'driver' } } // root driver inference = 100/50 + 50/30 + 20/10 = 170/90 tokens, $0.02. const rootChat = meteredChat([ { @@ -205,9 +270,8 @@ describe("driver inference metering — the driver's own tokens count against th // A sub-driver that meters turn 0 (40/20) then CRASHES (chat throws) on turn 1 — the crash // settles it `down`, which must STILL re-home the partial inference it durably metered. - const makeAgent = (raw: unknown): Agent => { - const p = raw as { kind?: string } - if (p?.kind === 'driver') { + const makeAgent = (profile: AgentProfile): Agent => { + if (profile.metadata?.kind === 'driver') { let t = 0 const crashingChat: ToolLoopChat = async () => { t += 1 @@ -237,7 +301,10 @@ describe("driver inference metering — the driver's own tokens count against th const rootChat = meteredChat([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'driver' }, task: 'go' } }, + { + name: 'spawn_agent', + arguments: { profile: { metadata: { kind: 'driver' } }, task: 'go' }, + }, ], usage: { input: 100, output: 50 }, }, @@ -327,6 +394,54 @@ describe("driver inference metering — the driver's own tokens count against th expect(n).toBe(3) }) + it.each([ + { + label: 'zero remaining child iterations', + rootBudget: { maxIterations: 0, maxTokens: 10_000 } satisfies Budget, + workerBudget: { maxIterations: 1, maxTokens: 100 } satisfies Budget, + }, + { + label: 'remaining capped dollars below the exact child allocation', + rootBudget: { maxIterations: 10, maxTokens: 10_000, maxUsd: 0.09 } satisfies Budget, + workerBudget: { maxIterations: 1, maxTokens: 100, maxUsd: 0.1 } satisfies Budget, + }, + ])('does not buy a manager turn with $label', async ({ rootBudget, workerBudget }) => { + const blobs = new InMemoryResultBlobStore() + const journal = new InMemorySpawnJournal() + let managerTurns = 0 + const result = await createSupervisor().run( + driverAgent({ + name: 'root', + brain: async () => { + managerTurns += 1 + return { + toolCalls: [{ id: 'would-spawn', name: 'spawn_agent', arguments: '{}' }], + usage: { input: 5, output: 5 }, + costUsd: 0.01, + } + }, + blobs, + makeWorkerAgent: () => workerLeaf('unused', { input: 1, output: 1 }), + perWorker: workerBudget, + systemPrompt: 'drive', + maxTurns: 0, + }), + 'cannot admit child work', + { + budget: rootBudget, + runId: `pre-turn-admission-${rootBudget.maxUsd ?? 'iterations'}`, + journal, + blobs, + executors: createExecutorRegistry(), + maxDepth: 2, + now: () => 0, + }, + ) + + expect(managerTurns).toBe(0) + expect(result.kind).toBe('no-winner') + }) + it('emits an agent.turn observability event per metered driver turn (the live A++ view)', async () => { const blobs = new InMemoryResultBlobStore() const journal = new InMemorySpawnJournal() diff --git a/tests/kernel/driver-recursion.test.ts b/tests/kernel/driver-recursion.test.ts index 21f3e931..81a3fae2 100644 --- a/tests/kernel/driver-recursion.test.ts +++ b/tests/kernel/driver-recursion.test.ts @@ -100,12 +100,13 @@ function scriptedDriver( scope: Scope, ) => Array<{ label: string; agent: Agent }>, observed: Observed, + childBudget = perChild, ): Agent { return { name, async act(task, scope: Scope): Promise { for (const c of spawnChildren(scope)) { - const res = scope.spawn(c.agent, task, { budget: perChild, label: c.label }) + const res = scope.spawn(c.agent, task, { budget: childBudget, label: c.label }) if (!res.ok) throw new Error(`${name}: spawn ${c.label} failed: ${res.reason}`) // The node id IS the nesting proof: a driver child's nested scope parents its own // children under the driver's node id, so the worker's id is `rec:s0:s0:s0` — three @@ -219,9 +220,8 @@ describe('recursive driver: agents drive agents drive agents', () => { ) expect(result.kind).toBe('winner') - // Sum spend over EVERY journaled tree (root + every nested tree). The conserved pool - // guarantees this never exceeds the root ceiling, because every spawn at every depth - // reserves from the SAME pool and fails closed when it can't cover the child. + // Sum spend over EVERY journaled tree (root + every nested tree). Each nested scope + // partitions its parent's reserved allocation and fails closed when a child cannot fit. const allTreeKeys = collectTreeKeys(journal) let totalTokens = 0 let totalIterations = 0 @@ -254,12 +254,9 @@ describe('recursive driver: agents drive agents drive agents', () => { } }) - it('budget is CONSERVED across depth: a deep spawn fails closed when the shared pool is too small', async () => { - // The root ceiling is sized to admit the mid driver's reservation but NOT the worker's - // on top of it — proving the nested scope reserves from the SAME conserved pool as the - // root. The mid driver's spawn of the worker fails closed (budget-exhausted), the driver - // throws, the parent types it into a down → no-winner. A non-shared pool would let the - // deep spawn succeed and the run would win — so this asserts conservation across depth. + it('budget is conserved across depth: a deep spawn cannot exceed its branch allocation', async () => { + // The root reserves 1000 tokens for the mid driver. Its nested scope owns exactly that + // partition, so a 1001-token child request fails closed even though no sibling is running. const journal = new InMemorySpawnJournal() const blobs = new InMemoryResultBlobStore() const observed = newObserved() @@ -269,14 +266,15 @@ describe('recursive driver: agents drive agents drive agents', () => { iterations: 1, score: 0.5, }) - const midDriver = scriptedDriver('mid', () => [{ label: 'w', agent: worker }], observed) + const midDriver = scriptedDriver('mid', () => [{ label: 'w', agent: worker }], observed, { + maxIterations: 4, + maxTokens: 1001, + }) const rootDriver = scriptedDriver( 'root', () => [{ label: 'mid', agent: driverChild('mid', midDriver, journal) }], observed, ) - // perChild reserves 1000 tokens / 4 iterations. The root pool holds room for exactly ONE - // such reservation (the mid driver); the worker's reservation on top must fail closed. const result = await createSupervisor().run( rootDriver, 'task', @@ -289,8 +287,7 @@ describe('recursive driver: agents drive agents drive agents', () => { ) expect(result.kind).toBe('no-winner') if (result.kind === 'no-winner') { - // The mid driver was reserved (root scope spawn), but the worker's nested spawn could - // not be covered by the remaining pool — the shared pool conserved across depth. + // The mid driver was reserved at the root, but its oversized nested child never started. expect(observed.spawnedIds).toContain('rec:s0') // mid driver reserved at the root expect(observed.spawnedIds).not.toContain('rec:s0:s0') // worker never admitted (no budget) } diff --git a/tests/kernel/durable-jsonl.test.ts b/tests/kernel/durable-jsonl.test.ts new file mode 100644 index 00000000..c4afee5d --- /dev/null +++ b/tests/kernel/durable-jsonl.test.ts @@ -0,0 +1,109 @@ +import { appendFile, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { writeAllBytes } from '../../src/durable/jsonl-file' +import { FileSpawnJournal } from '../../src/durable/spawn-journal' +import type { CoordinationEvent } from '../../src/mcp/tools/coordination' +import { FileCoordinationLog } from '../../src/runtime/supervise/coordination-log' + +describe('durable append-only JSONL', () => { + it('finishes short writes instead of acknowledging a truncated record', async () => { + const chunks: Buffer[] = [] + const handle = { + async write(buffer: Uint8Array, offset: number, length: number) { + const written = Math.min(3, length) + chunks.push(Buffer.from(buffer.subarray(offset, offset + written))) + return { bytesWritten: written, buffer } + }, + } + + await writeAllBytes(handle, 'abcdefgh') + + expect(Buffer.concat(chunks).toString('utf8')).toBe('abcdefgh') + expect(chunks.map((chunk) => chunk.length)).toEqual([3, 3, 2]) + }) + + it('recovers only an invalid unterminated final spawn-journal record', async () => { + const dir = await mkdtemp(join(tmpdir(), 'spawn-jsonl-tail-')) + try { + const path = join(dir, 'spawn.jsonl') + const journal = new FileSpawnJournal(path) + await journal.beginTree('run', '2026-07-29T00:00:00.000Z') + await appendFile(path, '{"kind":"event"') + + await expect(journal.loadTree('run')).resolves.toEqual([]) + await journal.beginTree('after-recovery', '2026-07-29T00:00:01.000Z') + await expect(journal.loadTree('run')).resolves.toEqual([]) + await expect(journal.loadTree('after-recovery')).resolves.toEqual([]) + + await writeFile( + path, + '{"kind":"begin","root":"run","at":"2026-07-29T00:00:00.000Z"}\n{bad}\n', + ) + await expect(journal.loadTree('run')).rejects.toThrow(/malformed JSONL record at line 2/) + + await writeFile( + path, + '{bad}\n{"kind":"begin","root":"run","at":"2026-07-29T00:00:00.000Z"}\n', + ) + await expect(journal.loadTree('run')).rejects.toThrow(/malformed JSONL record at line 1/) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + it('recovers only an invalid unterminated final coordination-log record', async () => { + const dir = await mkdtemp(join(tmpdir(), 'coordination-jsonl-tail-')) + try { + const path = join(dir, 'coordination.jsonl') + const log = new FileCoordinationLog(path) + const question: CoordinationEvent = { + type: 'question', + question: { + id: 'worker:q0', + from: 'worker', + level: 'worker', + question: 'Continue?', + reason: 'blocked', + urgency: 'blocks-run', + status: 'open', + openedAt: 0, + }, + } + await log.append('run', { seq: 0, at: 0, priority: 20, event: question }, 'owner') + await appendFile(path, '{"runId":"run"') + + await expect(log.load('run', 'owner')).resolves.toMatchObject({ + questions: [{ id: 'worker:q0', status: 'open' }], + }) + await log.append( + 'run', + { + seq: 1, + at: 1, + priority: 20, + event: { + ...question, + question: { ...question.question, id: 'worker:q1', question: 'Still continue?' }, + }, + }, + 'owner', + ) + await expect(log.load('run', 'owner')).resolves.toMatchObject({ + questions: [ + { id: 'worker:q0', status: 'open' }, + { id: 'worker:q1', status: 'open' }, + ], + }) + + await writeFile(path, '{bad}\n') + await expect(log.load('run', 'owner')).rejects.toThrow(/malformed JSONL record at line 1/) + + await writeFile(path, '{bad}\n{}\n') + await expect(log.load('run', 'owner')).rejects.toThrow(/malformed JSONL record at line 1/) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/kernel/inbox.test.ts b/tests/kernel/inbox.test.ts index 9ef2b851..9d19f2a0 100644 --- a/tests/kernel/inbox.test.ts +++ b/tests/kernel/inbox.test.ts @@ -1,6 +1,6 @@ import type { AgentProfile } from '@tangle-network/agent-interface' import { afterEach, describe, expect, it, vi } from 'vitest' -import { type AgentSpec, createExecutor, createInbox } from '../../src/runtime' +import { type AgentSpec, createBudgetPool, createExecutor, createInbox } from '../../src/runtime' describe('worker inbox (down-leg receive end)', () => { it('parses the down-message shapes; ignores malformed', () => { @@ -143,4 +143,70 @@ describe('router-tools executor drains the inbox', () => { ).toBe(true) expect(result.spent.iterations).toBe(1) }) + + it('marks dollar cost unknown for an unpriced model even when token usage is complete', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => noToolReply()), + ) + const factory = createExecutor({ + backend: 'router-tools', + model: 'unpriced-test-model', + routerBaseUrl: 'http://router.test', + routerKey: 'k', + tools: [], + executeToolCall: async () => '', + }) + const exec = factory( + { profile: { name: 'w' }, harness: null }, + { signal: new AbortController().signal, seams: {} }, + ) + + const result = await exec.execute('do the task', new AbortController().signal) + + expect(result.spent).toMatchObject({ + tokens: { input: 1, output: 1 }, + usd: 0, + usdKnown: false, + }) + const pool = createBudgetPool({ maxIterations: 2, maxTokens: 10, maxUsd: 1 }, () => 0) + const reservation = pool.reserve({ maxIterations: 1, maxTokens: 2, maxUsd: 1 }) + if (!reservation.ok) throw new Error('reservation should fit') + expect(() => pool.reconcile(reservation.ticket, result.spent)).toThrow(/unknown dollar cost/) + expect(pool.readout()).toMatchObject({ usdLeft: 0, usdKnown: false }) + }) + + it('marks dollar cost unknown for a priced model when token usage is missing', async () => { + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response(JSON.stringify({ choices: [{ message: { content: 'done' } }] }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ), + ) + const factory = createExecutor({ + backend: 'router-tools', + model: 'gpt-4o', + routerBaseUrl: 'http://router.test', + routerKey: 'k', + tools: [], + executeToolCall: async () => '', + }) + const exec = factory( + { profile: { name: 'w' }, harness: null }, + { signal: new AbortController().signal, seams: {} }, + ) + + const result = await exec.execute('do the task', new AbortController().signal) + + expect(result.spent).toMatchObject({ + tokens: { input: 0, output: 0 }, + tokensKnown: false, + usd: 0, + usdKnown: false, + }) + }) }) diff --git a/tests/kernel/supervise-convenience.test.ts b/tests/kernel/supervise-convenience.test.ts index 4174293f..333dfc93 100644 --- a/tests/kernel/supervise-convenience.test.ts +++ b/tests/kernel/supervise-convenience.test.ts @@ -89,6 +89,65 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ expect(result.kind).toBe('winner') }) + it('cascades the caller abort signal through the root and every live child', async () => { + const controller = new AbortController() + let started!: () => void + const childStarted = new Promise((resolve) => { + started = resolve + }) + let teardownCalled = false + const blockedLeaf = (): Agent => { + const executor: Executor = { + runtime: 'blocked-test-worker', + execute(_task, signal): Promise> { + started() + return new Promise((_, reject) => { + const abort = (): void => reject(new DOMException('aborted', 'AbortError')) + if (signal.aborted) abort() + else signal.addEventListener('abort', abort, { once: true }) + }) + }, + teardown: () => { + teardownCalled = true + return Promise.resolve({ destroyed: true }) + }, + resultArtifact: () => { + throw new Error('an aborted worker has no terminal artifact') + }, + } + const spec: AgentSpec = { + profile: { name: 'blocked-worker' } as AgentProfile, + harness: null, + executor, + } + return { + name: 'blocked-worker', + act: async () => undefined, + executorSpec: spec, + } as Agent & { executorSpec: AgentSpec } + } + const running = supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { + budget, + signal: controller.signal, + makeWorkerAgent: blockedLeaf, + brain: scriptedBrain([ + { + toolCalls: [ + { name: 'spawn_agent', arguments: { profile: { name: 'worker' }, task: 'go' } }, + ], + }, + { toolCalls: [{ name: 'await_event', arguments: {} }] }, + ]), + }) + + await childStarted + controller.abort() + const result = await running + + expect(result).toMatchObject({ kind: 'no-winner', reason: 'aborted' }) + expect(teardownCalled).toBe(true) + }) + it('runDir makes the run durable and resumable; unset stays in-memory', async () => { const dir = await mkdtemp(join(tmpdir(), 'supervise-rundir-')) try { diff --git a/tests/kernel/supervise.test.ts b/tests/kernel/supervise.test.ts index 9f2de820..69c8ba7c 100644 --- a/tests/kernel/supervise.test.ts +++ b/tests/kernel/supervise.test.ts @@ -144,6 +144,27 @@ async function beginScope(over: Partial[0]> = {}) // ── 1. Conserved budget pool ───────────────────────────────────────────────────── describe('conserved budget pool', () => { + it.each([ + { maxIterations: -1, maxTokens: 100 }, + { maxIterations: 1.5, maxTokens: 100 }, + { maxIterations: 1, maxTokens: -1 }, + { maxIterations: 1, maxTokens: Number.MAX_SAFE_INTEGER + 1 }, + { maxIterations: 1, maxTokens: 100, maxUsd: Number.POSITIVE_INFINITY }, + { maxIterations: 1, maxTokens: 100, deadlineMs: -1 }, + ] as Budget[])('rejects a malformed root budget before it can create capacity', (invalid) => { + expect(() => createBudgetPool(invalid, () => 0)).toThrow(/non-negative/) + }) + + it('rejects a negative reservation without changing the root balance', () => { + const pool = createBudgetPool({ maxIterations: 10, maxTokens: 100 }, () => 0) + expect(() => pool.reserve({ maxIterations: -10, maxTokens: -100 })).toThrow(/non-negative/) + expect(pool.readout()).toMatchObject({ + tokensLeft: 100, + tokensKnown: true, + reservedTokens: 0, + }) + }) + it('reserve fails closed when the pool cannot cover the child', () => { const pool = createBudgetPool({ maxIterations: 4, maxTokens: 1000 }, () => 0) const a = pool.reserve({ maxIterations: 2, maxTokens: 600, label: '' } as Budget) @@ -172,6 +193,25 @@ describe('conserved budget pool', () => { expect(pool.readout().reservedTokens).toBe(0) }) + it('records actual overspend and refuses later work instead of clamping telemetry', () => { + const pool = createBudgetPool({ maxIterations: 2, maxTokens: 20 }, () => 0) + const first = pool.reserve({ maxIterations: 1, maxTokens: 10 }) + if (!first.ok) throw new Error('reserve should have succeeded') + + pool.reconcile(first.ticket, { + iterations: 1, + tokens: { input: 20, output: 0 }, + usd: 0, + ms: 0, + }) + + expect(pool.readout().tokensLeft).toBe(0) + expect(pool.reserve({ maxIterations: 1, maxTokens: 1 })).toEqual({ + ok: false, + reason: 'budget-exhausted', + }) + }) + it('fails loud on a double reconcile (no silent double refund)', () => { const pool = createBudgetPool({ maxIterations: 10, maxTokens: 1000 }, () => 0) const r = pool.reserve({ maxIterations: 5, maxTokens: 800, label: '' } as Budget) @@ -218,6 +258,50 @@ describe('conserved budget pool', () => { expect(pool.readout().usdLeft).toBe(0) }) + it('preserves unknown dollar telemetry under an uncapped root without blocking admission', () => { + const pool = createBudgetPool({ maxIterations: 2, maxTokens: 1000 }, () => 0) + const r = pool.reserve({ maxIterations: 1, maxTokens: 500 }) + if (!r.ok) throw new Error('reserve should have succeeded') + + expect(() => + pool.reconcile(r.ticket, { + iterations: 1, + tokens: { input: 40, output: 60 }, + usd: 0, + usdKnown: false, + ms: 0, + }), + ).not.toThrow() + expect(pool.readout()).toMatchObject({ usdCapped: false, usdKnown: false, tokensLeft: 900 }) + expect(pool.reserve({ maxIterations: 1, maxTokens: 100 }).ok).toBe(true) + }) + + it('marks restored in-doubt dollar telemetry unknown even without a dollar limit', () => { + const pool = createBudgetPool({ maxIterations: 2, maxTokens: 1000 }, () => 0, { + uncertainReservations: [{ maxIterations: 1, maxTokens: 500 }], + }) + + expect(pool.readout()).toMatchObject({ + tokensKnown: false, + usdCapped: false, + usdKnown: false, + tokensLeft: 500, + }) + }) + + it('refuses malformed committed spend during restore instead of restoring it as zero', () => { + expect(() => + createBudgetPool({ maxIterations: 2, maxTokens: 1000 }, () => 0, { + committed: { + iterations: 1, + tokens: { input: -1, output: 0 }, + usd: 0, + ms: 0, + }, + }), + ).toThrow(/budget restore committed\.tokens\.input/) + }) + it('never interprets explicitly unknown dollar cost as $0 under a dollar limit', () => { const pool = createBudgetPool({ maxIterations: 2, maxTokens: 1000, maxUsd: 1 }, () => 0) const r = pool.reserve({ maxIterations: 1, maxTokens: 500, maxUsd: 1 } as Budget) @@ -231,6 +315,43 @@ describe('conserved budget pool', () => { ms: 0, }), ).toThrow(/unknown dollar cost/) + expect(pool.readout().usdLeft).toBe(0) + expect(() => pool.assertNoOpenTickets()).not.toThrow() + }) + + it('never interprets explicitly unknown token usage as zero', () => { + const pool = createBudgetPool({ maxIterations: 2, maxTokens: 1000 }, () => 0) + const r = pool.reserve({ maxIterations: 1, maxTokens: 500 }) + if (!r.ok) throw new Error('reserve should have succeeded') + expect(() => + pool.reconcile(r.ticket, { + iterations: 1, + tokens: { input: 0, output: 0 }, + tokensKnown: false, + usd: 0, + ms: 0, + }), + ).toThrow(/unknown token usage/) + expect(pool.readout()).toMatchObject({ tokensLeft: 0, tokensKnown: false }) + expect(pool.reserve({ maxIterations: 1, maxTokens: 1 })).toEqual({ + ok: false, + reason: 'budget-exhausted', + }) + expect(() => pool.assertNoOpenTickets()).not.toThrow() + }) + + it('exhausts live dollar capacity after observing unknown manager cost', () => { + const pool = createBudgetPool({ maxIterations: 2, maxTokens: 1000, maxUsd: 1 }, () => 0) + expect(() => + pool.observe({ + iterations: 1, + tokens: { input: 40, output: 60 }, + usd: 0, + usdKnown: false, + ms: 0, + }), + ).toThrow(/unknown dollar cost/) + expect(pool.readout()).toMatchObject({ tokensLeft: 900, usdLeft: 0, usdCapped: true }) }) it('spendFromUsageEvents folds tokens + usd on separate channels', () => { diff --git a/tests/kernel/supervisor-authoring.test.ts b/tests/kernel/supervisor-authoring.test.ts index a2ce5f74..16d0b7fe 100644 --- a/tests/kernel/supervisor-authoring.test.ts +++ b/tests/kernel/supervisor-authoring.test.ts @@ -57,8 +57,10 @@ describe('supervisor authoring — the supervisor DESIGNS each worker (profile), arguments: { profile: { name: 'parser', - systemPrompt: - 'You are a PARSER specialist. Tokenize the expression into numbers, operators and parens; emit a JSON token list. Validate balanced parens.', + prompt: { + systemPrompt: + 'You are a PARSER specialist. Tokenize the expression into numbers, operators and parens; emit a JSON token list. Validate balanced parens.', + }, }, task: 'parse the expression', }, @@ -72,9 +74,11 @@ describe('supervisor authoring — the supervisor DESIGNS each worker (profile), arguments: { profile: { name: 'evaluator', - systemPrompt: - 'You are an EVALUATOR specialist. Given a token list, apply operator precedence and compute the numeric result. Return only the number.', - model: 'deepseek-chat', + prompt: { + systemPrompt: + 'You are an EVALUATOR specialist. Given a token list, apply operator precedence and compute the numeric result. Return only the number.', + }, + model: { default: 'deepseek-chat' }, }, task: 'evaluate the tokens', }, @@ -91,7 +95,7 @@ describe('supervisor authoring — the supervisor DESIGNS each worker (profile), ] let n = 0 - const makeWorker = (raw: unknown): Agent => { + const makeWorker = (raw: AgentProfile): Agent => { const p = asAuthoredProfile(raw) if (p) authored.push(p) return deliveringLeaf(p?.name ?? `w${n++}`, { ok: true }) @@ -122,16 +126,18 @@ describe('supervisor authoring — the supervisor DESIGNS each worker (profile), expect(authored.length).toBe(2) expect(authored[0]!.name).toBe('parser') expect(authored[1]!.name).toBe('evaluator') - expect(authored[0]!.systemPrompt).not.toBe(authored[1]!.systemPrompt) - expect(authored[0]!.systemPrompt).toContain('PARSER') - expect(authored[1]!.model).toBe('deepseek-chat') // the supervisor also chose the model per sub-task + expect(authored[0]!.prompt.systemPrompt).not.toBe(authored[1]!.prompt.systemPrompt) + expect(authored[0]!.prompt.systemPrompt).toContain('PARSER') + expect(authored[1]!.model?.default).toBe('deepseek-chat') }) it('rejects an empty/placeholder profile (a skill violation the system can catch)', () => { expect(asAuthoredProfile({})).toBeNull() expect(asAuthoredProfile({ systemPrompt: '' })).toBeNull() expect(asAuthoredProfile({ systemPrompt: ' ' })).toBeNull() - expect(asAuthoredProfile({ name: 'w', systemPrompt: 'real instructions' })?.name).toBe('w') + expect( + asAuthoredProfile({ name: 'w', prompt: { systemPrompt: 'real instructions' } })?.name, + ).toBe('w') }) it('the skill is the supervisor prompt and demands authored (non-empty) profiles', () => { diff --git a/tests/knowledge-supervised-update.test.ts b/tests/knowledge-supervised-update.test.ts index f73faec8..885f18a9 100644 --- a/tests/knowledge-supervised-update.test.ts +++ b/tests/knowledge-supervised-update.test.ts @@ -75,7 +75,7 @@ describe('knowledge supervisor integration', () => { expect(captured?.task).toContain('Goal: candidate goal') expect(captured?.task).toContain('Knowledge base root: /kb/candidate') expect(captured?.profile.name).toBe('knowledge-research-supervisor') - expect(captured?.profile.systemPrompt).toContain( + expect(captured?.profile.prompt?.systemPrompt).toContain( 'Each researcher worker you spawn follows this contract', ) }) diff --git a/tests/mcp/delegate.test.ts b/tests/mcp/delegate.test.ts index 7a5339bc..94a73a8c 100644 --- a/tests/mcp/delegate.test.ts +++ b/tests/mcp/delegate.test.ts @@ -109,8 +109,8 @@ describe('delegate MCP tool — generic delegation verb that returns cost', () = it('applies a per-call model override', async () => { const handler = createDelegateHandler({ router, backend, model: 'deepseek-v4-flash' }) await handler({ intent: 'do x', model: 'glm-5.2' }) - const [profile] = superviseSpy.mock.calls[0] as [{ model?: string }] - expect(profile.model).toBe('glm-5.2') + const [profile] = superviseSpy.mock.calls[0] as [{ model?: { default?: string } }] + expect(profile.model?.default).toBe('glm-5.2') }) it('createMcpServer registers `delegate` only when delegateSupervisor is wired', () => { diff --git a/tests/runtime/bridge-executor.test.ts b/tests/runtime/bridge-executor.test.ts index 062811dc..e6e83f72 100644 --- a/tests/runtime/bridge-executor.test.ts +++ b/tests/runtime/bridge-executor.test.ts @@ -1,7 +1,9 @@ import { PassThrough, type Readable } from 'node:stream' import type { SandboxEvent } from '@tangle-network/sandbox' import { afterEach, describe, expect, it, vi } from 'vitest' -import { createExecutor, inlineSandboxClient } from '../../src/runtime' +import { createExecutor, type ExecutorConfig, inlineSandboxClient } from '../../src/runtime' +import { workerFromBackend } from '../../src/runtime/supervise/supervise' +import type { Agent, AgentSpec, UsageEvent } from '../../src/runtime/supervise/types' // `bridgeExecutor` POSTs each turn over the `node:http` core client, not global // `fetch`: the bridge runs a harness CLI and streams only once it starts @@ -25,8 +27,17 @@ vi.mock('node:http', async () => { end: () => { const payload = JSON.parse(body || '{}') as Record if (!bridgeHttpHandler) throw new Error('bridgeHttpHandler not set') - const res = bridgeHttpHandler(payload) as Readable & { statusCode?: number } + const res = bridgeHttpHandler(payload) as Readable & { + statusCode?: number + headers?: Record + } res.statusCode = res.statusCode ?? 200 + if (res.statusCode >= 200 && res.statusCode < 300) { + res.headers = { + 'x-run-id': String(payload.run_id), + 'x-run-request-digest': `sha256:${'a'.repeat(64)}`, + } + } cb(res) }, on: () => {}, @@ -40,6 +51,7 @@ function sse(content: string, input: number, output: number): Readable { const stream = new PassThrough() stream.end( [ + 'id: 1', `data: ${JSON.stringify({ choices: [{ delta: { content } }], usage: { prompt_tokens: input, completion_tokens: output }, @@ -147,6 +159,158 @@ describe('bridgeExecutor over node:http', () => { expect(seen[0]?.model).toBe('kimi-code/kimi-k2.6') }) + it('captures model and nested profile policy when createExecutor is called', async () => { + const seen: Array> = [] + bridgeHttpHandler = (payload) => { + seen.push(payload) + return sse('ok', 1, 2) + } + const config: Extract = { + backend: 'bridge', + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'safe-model', + agentProfile: { + name: 'policy-overlay', + permissions: { shell: 'deny' }, + }, + } + const client = inlineSandboxClient(createExecutor(config)) + config.model = 'mutated-model' + if (config.agentProfile?.permissions) config.agentProfile.permissions.shell = 'allow' + + await runOnce(client, 'go') + + expect(seen[0]?.model).toBe('safe-model') + expect(seen[0]?.agent_profile).toMatchObject({ permissions: { shell: 'deny' } }) + }) + + it('captures the turn limit before callers can expand the execution budget', async () => { + let requests = 0 + let deliver: (message: unknown) => void = () => {} + bridgeHttpHandler = () => { + requests += 1 + if (requests === 1) deliver({ steer: 'run another turn' }) + return sse(`turn-${requests}`, 1, 1) + } + const config: Extract = { + backend: 'bridge', + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'safe-model', + maxTurns: 1, + } + const factory = createExecutor(config) + config.maxTurns = 3 + const executor = factory( + { profile: { name: 'budget-worker' }, harness: null }, + { signal: new AbortController().signal, seams: {} }, + ) + deliver = (message) => executor.deliver?.(message) + const run = executor.execute('go', new AbortController().signal) + if (!isUsageStream(run)) throw new Error('bridge worker must stream usage') + for await (const _event of run) { + // drain the bridge stream + } + + expect(requests).toBe(1) + expect(executor.resultArtifact().spent.iterations).toBe(1) + }) + + it('gives parallel reusable workers isolated bridge sessions', async () => { + const seen: Array> = [] + bridgeHttpHandler = (payload) => { + seen.push(payload) + return sse('ok', 1, 2) + } + const make = workerFromBackend({ + backend: 'bridge', + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'safe-model', + }) + const workers = ['a', 'b'].map( + (name, index) => + make( + { name }, + { + assignmentId: `ordinal:${index}`, + budget: { maxIterations: 1, maxTokens: 100 }, + task: 'go', + label: name, + }, + ) as Agent & { + executorSpec: AgentSpec + }, + ) + const executors = workers.map((worker) => + worker.executorSpec.executorFactory?.(worker.executorSpec, { + signal: new AbortController().signal, + seams: {}, + }), + ) + + await Promise.all( + executors.map(async (executor) => { + if (!executor) throw new Error('worker executor factory missing') + const run = executor.execute('go', new AbortController().signal) + if (!isUsageStream(run)) throw new Error('bridge worker must stream usage') + for await (const _event of run) { + // drain the bridge stream + } + }), + ) + + expect(seen).toHaveLength(2) + const sessions = seen.map((request) => request.session_id) + expect(sessions.every((session) => typeof session === 'string')).toBe(true) + expect(new Set(sessions).size).toBe(2) + }) + + it('reconstructs the same bridge session for the same durable worker assignment', async () => { + const seen: Array> = [] + bridgeHttpHandler = (payload) => { + seen.push(payload) + return sse('ok', 1, 2) + } + const backend = { + backend: 'bridge' as const, + bridgeUrl: 'http://bridge.test', + bridgeBearer: 'secret', + model: 'safe-model', + } + const context = { + assignmentId: 'key:stable-experiment', + budget: { maxIterations: 1, maxTokens: 100 }, + task: 'go', + label: 'stable-experiment', + key: 'stable-experiment', + } + const workers = [workerFromBackend(backend), workerFromBackend(backend)].map( + (make) => + make({ name: 'worker' }, context) as Agent & { + executorSpec: AgentSpec + }, + ) + + for (const worker of workers) { + const executor = worker.executorSpec.executorFactory?.(worker.executorSpec, { + signal: new AbortController().signal, + seams: {}, + }) + if (!executor) throw new Error('worker executor factory missing') + const run = executor.execute('go', new AbortController().signal) + if (!isUsageStream(run)) throw new Error('bridge worker must stream usage') + for await (const _event of run) { + // drain the bridge stream + } + } + + expect(seen).toHaveLength(2) + expect(seen[0]?.session_id).toMatch(/^supervised-worker-[a-f0-9]{64}$/) + expect(seen[1]?.session_id).toBe(seen[0]?.session_id) + }) + it('throws on a non-2xx bridge response', async () => { bridgeHttpHandler = () => { const s = new PassThrough() as PassThrough & { statusCode?: number } @@ -157,3 +321,11 @@ describe('bridgeExecutor over node:http', () => { await expect(runOnce(bridgeClient('kimi-code/k2'), 'go')).rejects.toThrow(/bridge 500/) }) }) + +function isUsageStream(value: unknown): value is AsyncIterable { + return ( + value !== null && + typeof value === 'object' && + typeof (value as { [Symbol.asyncIterator]?: unknown })[Symbol.asyncIterator] === 'function' + ) +} diff --git a/tests/runtime/executor-profile-model.test.ts b/tests/runtime/executor-profile-model.test.ts new file mode 100644 index 00000000..a634b9ff --- /dev/null +++ b/tests/runtime/executor-profile-model.test.ts @@ -0,0 +1,83 @@ +import { createServer, type Server } from 'node:http' +import type { AddressInfo } from 'node:net' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { afterEach, describe, expect, it } from 'vitest' +import { type AgentSpec, createExecutor } from '../../src/runtime' + +const profile: AgentProfile = { + name: 'profile-model-worker', + model: { default: 'profile-selected-model' }, +} + +const spec: AgentSpec = { profile, harness: null } + +let server: Server | undefined + +async function startRouter(onRequest: (body: Record) => void): Promise { + server = createServer(async (request, response) => { + const chunks: Buffer[] = [] + for await (const chunk of request) chunks.push(Buffer.from(chunk)) + onRequest(JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record) + response.writeHead(200, { 'content-type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [{ message: { content: 'done', tool_calls: [] } }], + usage: { prompt_tokens: 3, completion_tokens: 2 }, + }), + ) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + return `http://127.0.0.1:${port}` +} + +describe('router executor model precedence', () => { + afterEach(async () => { + if (server) await new Promise((resolve) => server?.close(() => resolve())) + server = undefined + }) + + it('uses AgentProfile.model.default instead of the router fallback', async () => { + let request: Record | undefined + const routerBaseUrl = await startRouter((body) => { + request = body + }) + const factory = createExecutor({ + backend: 'router', + routerBaseUrl, + routerKey: 'key', + model: 'backend-fallback-model', + }) + const executor = factory(spec, { + signal: new AbortController().signal, + seams: {}, + }) + + await executor.execute('do the task', new AbortController().signal) + + expect(request?.model).toBe('profile-selected-model') + }) + + it('uses AgentProfile.model.default instead of the router-tools fallback', async () => { + let request: Record | undefined + const routerBaseUrl = await startRouter((body) => { + request = body + }) + const factory = createExecutor({ + backend: 'router-tools', + routerBaseUrl, + routerKey: 'key', + model: 'backend-fallback-model', + tools: [], + executeToolCall: async () => '', + }) + const executor = factory(spec, { + signal: new AbortController().signal, + seams: {}, + }) + + await executor.execute('do the task', new AbortController().signal) + + expect(request?.model).toBe('profile-selected-model') + }) +}) diff --git a/tests/runtime/mid-flight-steering.test.ts b/tests/runtime/mid-flight-steering.test.ts index 4944e30f..21812a03 100644 --- a/tests/runtime/mid-flight-steering.test.ts +++ b/tests/runtime/mid-flight-steering.test.ts @@ -21,6 +21,9 @@ * post-steer actions never change. */ +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' import { describe, expect, it } from 'vitest' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' @@ -32,6 +35,7 @@ import type { SandboxClient } from '../../src/runtime/types' const WRONG = 'legacy/wrong.ts' const RIGHT = 'core/right.ts' const STEER = `stop editing ${WRONG} — the change belongs in ${RIGHT}` +const ANSWER = `continue in ${RIGHT}` const budget: Budget = { maxIterations: 200, maxTokens: 400_000 } @@ -180,6 +184,54 @@ function steeringBrain(harness: FakeHarness, record: BrainRecord): ToolLoopChat } } +function missingMessageAuthorityBrain( + harness: FakeHarness, + record: { steer?: Record; answer?: Record }, +): ToolLoopChat { + let turn = 0 + let workerId = 'w' + let questionId = 'q' + return async (messages) => { + const lastTool = [...messages] + .reverse() + .find((message) => (message as { role?: string }).role === 'tool') as + | { content?: string } + | undefined + const parsed = lastTool?.content ? safeJson(lastTool.content) : undefined + turn += 1 + + if (turn === 1) { + return call('spawn_agent', { profile: { name: 'coder' }, task: 'make the change' }) + } + if (turn === 2) { + workerId = String(parsed?.workerId ?? workerId) + await harness.workingOnWrongFile + return call('steer_agent', { workerId, instruction: STEER }) + } + if (turn === 3) { + record.steer = parsed + return call('ask_parent', { + from: workerId, + level: 'worker', + question: 'Which file should I edit?', + reason: 'Two plausible targets', + urgency: 'blocks-step', + }) + } + if (turn === 4) { + const question = parsed?.question as Record | undefined + questionId = String(question?.id ?? questionId) + return call('answer_question', { questionId, answer: ANSWER }) + } + if (turn === 5) { + record.answer = parsed + harness.releaseFirstTurn() + return call('await_event', {}) + } + return { toolCalls: [], content: 'done' } + } +} + function call(name: string, args: Record) { return { toolCalls: [{ id: `${name}-1`, name, arguments: JSON.stringify(args) }], @@ -206,23 +258,94 @@ function backend(harness: FakeHarness, steerable: boolean): ExecutorConfig { } as ExecutorConfig } -async function runSupervisedSteer(steerable: boolean) { +interface AuthorityRecord { + spawnCalls: number + messages: Array<{ instruction: string; frozen: boolean; hasIdentity: boolean }> +} + +async function runSupervisedSteer(steerable: boolean, authority?: AuthorityRecord) { const harness = createFakeHarness() const record: BrainRecord = {} const result = await supervise( - { name: 'root', harness: null, systemPrompt: 'drive one coder and correct it' }, + { + name: 'root', + harness: 'cli-base', + prompt: { systemPrompt: 'drive one coder and correct it' }, + }, 'change the right module', { budget, backend: backend(harness, steerable), brain: steeringBrain(harness, record), maxTurns: 8, + ...(authority + ? { + authorizeSpawn(input) { + authority.spawnCalls += 1 + return { profile: input.profile } + }, + authorizeMessage(input) { + authority.messages.push({ + instruction: input.instruction, + frozen: Object.isFrozen(input) && Object.isFrozen(input.workerIdentity), + hasIdentity: + input.workerIdentity.profileDigest !== undefined && + input.workerIdentity.taskDigest !== undefined, + }) + return { instruction: input.instruction } + }, + } + : {}), }, ) return { harness, record, result } } describe('mid-flight steering — a supervisor observes a live worker and changes what it does', () => { + it('refuses steer and answer when spawn authority has no message authority', async () => { + const runDir = await mkdtemp(join(tmpdir(), 'supervise-message-authority-')) + const harness = createFakeHarness() + const record: { steer?: Record; answer?: Record } = {} + try { + await supervise({ name: 'root', harness: 'cli-base' }, 'change the right module', { + budget, + backend: backend(harness, true), + brain: missingMessageAuthorityBrain(harness, record), + runDir, + runId: 'message-authority-refusal', + authorizeSpawn: (input) => ({ profile: input.profile }), + }) + + expect(JSON.stringify(record.steer)).toContain('authorizeMessage is required') + expect(JSON.stringify(record.answer)).toContain('authorizeMessage is required') + expect( + harness.prompts.some((prompt) => prompt.includes(STEER) || prompt.includes(ANSWER)), + ).toBe(false) + const coordinationLog = await readFile(join(runDir, 'coordination-log.jsonl'), 'utf8') + const eventTypes = coordinationLog + .trim() + .split('\n') + .filter(Boolean) + .map((line) => (JSON.parse(line) as { event: { type: string } }).event.type) + expect(eventTypes).not.toContain('instruction') + } finally { + harness.releaseFirstTurn() + await rm(runDir, { recursive: true, force: true }) + } + }) + + it('authorizes the continuation against the exact live worker identity', { + timeout: 30_000, + }, async () => { + const authority: AuthorityRecord = { spawnCalls: 0, messages: [] } + const { record } = await runSupervisedSteer(true, authority) + expect(record.steerResult?.delivered).toBe(true) + expect(authority).toEqual({ + spawnCalls: 1, + messages: [{ instruction: STEER, frozen: true, hasIdentity: true }], + }) + }) + it('delivers a steer to a RUNNING sandbox worker and the worker acts differently afterwards', { timeout: 30_000, }, async () => { diff --git a/tests/runtime/pi-executor.test.ts b/tests/runtime/pi-executor.test.ts index 3ae65058..9c75e151 100644 --- a/tests/runtime/pi-executor.test.ts +++ b/tests/runtime/pi-executor.test.ts @@ -29,6 +29,7 @@ const FAKE_PI = `#!/usr/bin/env node const fs = require('node:fs') const log = process.env.PI_COMMAND_LOG const emit = (o) => process.stdout.write(JSON.stringify(o) + '\\n') +fs.appendFileSync(log, JSON.stringify({ type: 'argv', args: process.argv.slice(2) }) + '\\n') let buf = '' let turn = 0 process.stdin.on('data', (c) => { @@ -50,7 +51,21 @@ process.stdin.on('data', (c) => { emit({ type: 'turn_start' }) emit({ type: 'tool_execution_start', toolCallId: 't' + myTurn, toolName: 'edit', args: { path: target } }) emit({ type: 'tool_execution_end', toolCallId: 't' + myTurn, toolName: 'edit', result: 'ok', isError: false, args: { path: target } }) - emit({ type: 'turn_end', message: { role: 'assistant', content: 'edited ' + target, usage: { input: 30, output: 12, cost: 0.002 } }, toolResults: [] }) + const message = { + role: 'assistant', + content: [{ type: 'text', text: 'edited ' + target }], + usage: { + input: 30, + output: 12, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 42, + cost: { input: 0.001, output: 0.001, cacheRead: 0, cacheWrite: 0, total: 0.002 } + }, + timestamp: Date.now() + } + emit({ type: 'message_end', message }) + emit({ type: 'turn_end', message, toolResults: [] }) emit({ type: 'agent_end', messages: [] }) } }) @@ -98,7 +113,38 @@ async function readCommands(): Promise>> { } describe('piExecutor — pi wrapped, not forked', () => { - it('runs a turn, reports REAL usage off pi events, and exposes live progress + tool spans', async () => { + it('uses AgentProfile.model.default instead of the backend fallback', async () => { + await writeFile(commandLog, '') + const ctx = piCtx() + const withFallback: ExecutorContext = { + ...ctx, + seams: { + ...ctx.seams, + [piSeamKey]: { + ...(ctx.seams[piSeamKey] as Record), + model: 'fallback/ignored-model', + }, + }, + } + const ex = piExecutor( + { + profile: { + name: 'profile-model', + model: { default: 'profile/selected-model' }, + }, + harness: null, + }, + withFallback, + ) + + await drain(ex.execute('make the change', withFallback.signal) as AsyncIterable) + const argv = (await readCommands()).find((command) => command.type === 'argv')?.args + + expect(argv).toEqual(['--mode', 'rpc', '--provider', 'profile', '--model', 'selected-model']) + await ex.teardown('brutalKill') + }) + + it('counts duplicated message_end + turn_end telemetry once from authoritative turn_end', async () => { await writeFile(commandLog, '') const ctx = piCtx() const ex = piExecutor(spec, ctx) @@ -109,8 +155,10 @@ describe('piExecutor — pi wrapped, not forked', () => { ) // REAL usage only — the numbers the fake pi reported, not a fabricated estimate. - expect(events).toContainEqual({ kind: 'tokens', input: 30, output: 12 }) - expect(events).toContainEqual({ kind: 'cost', usd: 0.002 }) + expect(events.filter((event) => event.kind === 'tokens')).toEqual([ + { kind: 'tokens', input: 30, output: 12 }, + ]) + expect(events.filter((event) => event.kind === 'cost')).toEqual([{ kind: 'cost', usd: 0.002 }]) expect(events.filter((e) => e.kind === 'iteration')).toHaveLength(1) const progress = ex.progress?.() @@ -123,6 +171,9 @@ describe('piExecutor — pi wrapped, not forked', () => { const artifact = ex.resultArtifact() expect(String((artifact.out as { content: string }).content)).toContain('edited wrong.ts') expect(artifact.spent.tokens).toEqual({ input: 30, output: 12 }) + expect(artifact.spent.usd).toBe(0.002) + expect(artifact.spent).not.toHaveProperty('tokensKnown') + expect(artifact.spent).not.toHaveProperty('usdKnown') await ex.teardown('brutalKill') }) diff --git a/tests/runtime/resume-aware-driver.test.ts b/tests/runtime/resume-aware-driver.test.ts index ac345119..dc67c6ef 100644 --- a/tests/runtime/resume-aware-driver.test.ts +++ b/tests/runtime/resume-aware-driver.test.ts @@ -135,12 +135,13 @@ describe('resume-aware built-in driver — a killed coordinator resumes without expect(resumed.kind).toBe('winner') expect(resumed.settledNodes).toEqual(['w1', 'w2', 'w3', 'w4', 'w5']) - // ── The real bar: same output, same worker spend as never having crashed ────────────── + // The output matches control. Spend does not pretend the two killed executions were free: + // their missing receipts are charged at both declared ceilings and marked unknown. expect(resumed.out).toBe(controlReport.out) - expect(resumed.spentBreakdown?.childWork).toEqual(controlReport.spentBreakdown?.childWork) - // Stated absolutely, not just relatively: five workers' worth of paid work, once each. - expect(resumed.spentBreakdown?.childWork.iterations).toBe(5) - expect(resumed.spentBreakdown?.childWork.tokens).toEqual({ input: 50, output: 50 }) + expect(resumed.spentBreakdown?.childWork.iterations).toBe(15) + expect(resumed.spentBreakdown?.childWork.tokens).toEqual({ input: 20_050, output: 50 }) + expect(resumed.spentBreakdown?.childWork.tokensKnown).toBe(false) + expect(resumed.spentBreakdown?.childWork.usdKnown).toBe(false) expect(resumed.spentBreakdown?.childWork.usd).toBeCloseTo(0.05, 10) // The ONLY channel that differs is the coordinator's own inference: a restarted coordinator diff --git a/tests/runtime/spawn-keys.test.ts b/tests/runtime/spawn-keys.test.ts index 0b35b37b..9707fcf2 100644 --- a/tests/runtime/spawn-keys.test.ts +++ b/tests/runtime/spawn-keys.test.ts @@ -136,7 +136,34 @@ describe('semantic spawn keys', () => { }) expect(result.kind).toBe('winner') expect(seen.reason).toBe('duplicate-key') - expect(seen.runsAtRefusal).toBe(1) + // The first execution waits for its identity append, so the duplicate can be refused before + // either worker body starts. The admitted original still runs exactly once afterward. + expect(seen.runsAtRefusal).toBe(0) + expect(runs.n).toBe(1) + }) + + it('refuses a completed key when the requested profile or task changes', async () => { + const runs = { n: 0 } + const seen: Record = {} + await runRoot(async (_task, scope) => { + const first = scope.spawn(countingLeaf('original', 'A', runs), 'task A', { + budget: childBudget, + label: 'assignment', + key: 'same-key', + }) + expect(first.ok).toBe(true) + await scope.next() + const freeBefore = scope.budget.tokensLeft + const changed = scope.spawn(countingLeaf('changed', 'B', runs), 'task B', { + budget: childBudget, + label: 'assignment', + key: 'same-key', + }) + seen.reason = changed.ok ? 'accepted' : changed.reason + seen.freeUnchanged = scope.budget.tokensLeft === freeBefore + return 'done' + }) + expect(seen).toEqual({ reason: 'key-conflict', freeUnchanged: true }) expect(runs.n).toBe(1) }) diff --git a/tests/runtime/stop-rules.test.ts b/tests/runtime/stop-rules.test.ts index 0263fe50..2cc61037 100644 --- a/tests/runtime/stop-rules.test.ts +++ b/tests/runtime/stop-rules.test.ts @@ -353,7 +353,10 @@ describe('driverAgent stopRule — evaluated after the hard ceilings, never inst const chat = scriptedBrain([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'worker' }, task: 'go' } }, + { + name: 'spawn_agent', + arguments: { profile: { metadata: { kind: 'worker' } }, task: 'go' }, + }, { name: 'await_event', arguments: {} }, ], }, @@ -424,7 +427,10 @@ describe('driverAgent stopRule — evaluated after the hard ceilings, never inst brain: scriptedBrain([ { toolCalls: [ - { name: 'spawn_agent', arguments: { profile: { kind: 'worker' }, task: 'go' } }, + { + name: 'spawn_agent', + arguments: { profile: { metadata: { kind: 'worker' } }, task: 'go' }, + }, { name: 'await_event', arguments: {} }, ], }, diff --git a/tests/runtime/supervisor-finalizer.test.ts b/tests/runtime/supervisor-finalizer.test.ts index 4f4506e1..8fb42728 100644 --- a/tests/runtime/supervisor-finalizer.test.ts +++ b/tests/runtime/supervisor-finalizer.test.ts @@ -253,7 +253,7 @@ const makeWorker = (profile: unknown) => { describe('SupervisorFinalizer — end to end through supervise()', () => { it('the default keeps the delivered answer over a higher-scoring unchecked one', async () => { - const result = await supervise({ name: 'root', harness: null }, 'task', { + const result = await supervise({ name: 'root', harness: 'cli-base' }, 'task', { budget, perWorker: { maxIterations: 5, maxTokens: 10_000 }, makeWorkerAgent: makeWorker, @@ -264,7 +264,7 @@ describe('SupervisorFinalizer — end to end through supervise()', () => { }) it('an opted-in collectDelivered changes the SHAPE without ever widening eligibility', async () => { - const result = await supervise({ name: 'root', harness: null }, 'task', { + const result = await supervise({ name: 'root', harness: 'cli-base' }, 'task', { budget, perWorker: { maxIterations: 5, maxTokens: 10_000 }, makeWorkerAgent: makeWorker, @@ -278,7 +278,7 @@ describe('SupervisorFinalizer — end to end through supervise()', () => { }) it('a run whose only high scorer is unchecked is a no-winner, not a rescued output', async () => { - const result = await supervise({ name: 'root', harness: null }, 'task', { + const result = await supervise({ name: 'root', harness: 'cli-base' }, 'task', { budget, perWorker: { maxIterations: 5, maxTokens: 10_000 }, makeWorkerAgent: () => leaf('unchecked', 'UNCHECKED-PROSE', 0.99, false), diff --git a/tests/runtime/wait-states.test.ts b/tests/runtime/wait-states.test.ts index a0b97e2e..cf371a8a 100644 --- a/tests/runtime/wait-states.test.ts +++ b/tests/runtime/wait-states.test.ts @@ -18,7 +18,13 @@ import { } from '../../src/durable/spawn-journal' import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' import { createSupervisor } from '../../src/runtime/supervise/supervisor' -import type { Agent, Scope, Settled, SpawnEvent } from '../../src/runtime/supervise/types' +import type { + Agent, + Scope, + Settled, + SpawnEvent, + SpawnJournal, +} from '../../src/runtime/supervise/types' import { createWaitProbes, isWaitOutcome, @@ -227,7 +233,7 @@ describe('wait-states', () => { expect(scope.view.nodes).toHaveLength(0) return 'refused' }, - { probes: { known: () => true }, deadlineMs: Date.now() + 1_000 }, + { probes: { known: () => true }, deadlineMs: 1_000 }, ) }) @@ -270,4 +276,101 @@ describe('wait-states', () => { const cursors = events.filter((e) => e.kind === 'woken').map((e) => e.seq) expect(new Set(cursors).size).toBe(cursors.length) }) + + it('commits a fresh wait before its timer can wake', async () => { + const base = new InMemorySpawnJournal() + const appendStarted = deferred() + const releaseWaiting = deferred() + const appendAttempts: SpawnEvent['kind'][] = [] + const journal: SpawnJournal = { + loadTree: (root) => base.loadTree(root), + beginTree: (root, at) => base.beginTree(root, at), + async appendEvent(root, event): Promise { + appendAttempts.push(event.kind) + if (event.kind === 'waiting') { + appendStarted.resolve() + await releaseWaiting.promise + } + await base.appendEvent(root, event) + }, + } + const root: Agent = { + name: 'commit-wait-first', + async act(_task, scope): Promise { + const armed = scope.wait({ kind: 'timer', untilMs: Date.now() }, { label: 'now' }) + expect(armed.ok).toBe(true) + expect((await scope.next())?.kind).toBe('done') + return 'woke' + }, + } + + const running = createSupervisor().run(root, 'task', { + budget: { maxIterations: 1, maxTokens: 1 }, + runId: 'commit-wait-first', + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }) + await appendStarted.promise + await new Promise((resolve) => setImmediate(resolve)) + const attemptsBeforeCommit = [...appendAttempts] + releaseWaiting.resolve() + const result = await running + + expect(attemptsBeforeCommit.filter((kind) => kind === 'waiting' || kind === 'woken')).toEqual([ + 'waiting', + ]) + expect(result.kind).toBe('winner') + expect(appendAttempts.filter((kind) => kind === 'waiting' || kind === 'woken')).toEqual([ + 'waiting', + 'woken', + ]) + }) + + it('does not journal a wake when the wait arm itself was never committed', async () => { + const base = new InMemorySpawnJournal() + const appendAttempts: SpawnEvent['kind'][] = [] + const journal: SpawnJournal = { + loadTree: (root) => base.loadTree(root), + beginTree: (root, at) => base.beginTree(root, at), + async appendEvent(root, event): Promise { + appendAttempts.push(event.kind) + if (event.kind === 'waiting') throw new Error('journal unavailable') + await base.appendEvent(root, event) + }, + } + + const result = await createSupervisor().run( + { + name: 'failed-wait-arm', + async act(_task, scope): Promise { + const armed = scope.wait({ kind: 'timer', untilMs: Date.now() }, { label: 'now' }) + expect(armed.ok).toBe(true) + const settled = await scope.next() + expect(settled?.kind).toBe('down') + return settled?.kind === 'down' ? settled.reason : 'unexpected' + }, + }, + 'task', + { + budget: { maxIterations: 1, maxTokens: 1 }, + runId: 'failed-wait-arm', + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + }, + ) + + expect(result.kind).toBe('winner') + if (result.kind === 'winner') expect(result.out).toContain('journal unavailable') + expect(appendAttempts).not.toContain('woken') + }) }) + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} diff --git a/tests/supervisor-loop-example.test.ts b/tests/supervisor-loop-example.test.ts index dc5c083f..c7fe3b1d 100644 --- a/tests/supervisor-loop-example.test.ts +++ b/tests/supervisor-loop-example.test.ts @@ -65,8 +65,10 @@ describe('supervisor-loop example — supervise() on the scripted brain (offline const result = await supervise( { name: 'supervisor', - harness: null, - systemPrompt: 'You are a supervisor. Spawn a worker, await it, and stop on delivery.', + harness: 'cli-base', + prompt: { + systemPrompt: 'You are a supervisor. Spawn a worker, await it, and stop on delivery.', + }, }, demoGoal, { From 2242bb594738e9f645829a583645ead1e3124ed6 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 10:55:01 -0600 Subject: [PATCH 10/12] feat(supervise): add recursive product control and durable identity --- CHANGELOG.md | 8 + docs/api/candidate-execution.md | 18 + docs/api/index.md | 68 +- docs/api/primitive-catalog.md | 30 +- docs/api/runtime.md | 635 +++++++++++++++--- docs/api/runtime/environment-provider.md | 2 +- docs/canonical-api.md | 18 +- package.json | 6 +- pnpm-lock.yaml | 91 ++- pnpm-workspace.yaml | 8 +- src/candidate-execution/index.ts | 3 + src/candidate-execution/profile.ts | 2 + src/durable/spawn-journal.ts | 163 +++++ src/runtime/index.ts | 12 + src/runtime/supervise/coordination-driver.ts | 14 + src/runtime/supervise/driver-executor.ts | 54 +- src/runtime/supervise/inbox.ts | 11 +- src/runtime/supervise/scope.ts | 15 +- src/runtime/supervise/supervise.ts | 235 ++++++- src/runtime/supervise/supervisor-agent.ts | 44 +- src/runtime/supervise/supervisor.ts | 591 ++++++++-------- src/runtime/supervise/tree-key.ts | 31 + src/runtime/supervise/types.ts | 32 +- .../fixtures/agent-improvement-proposal.json | 10 +- .../agent-profile-improvement-proposal.json | 4 +- ...candidate-execution-export-surface.test.ts | 26 + tests/kernel/driver-recursion.test.ts | 103 +++ tests/kernel/inbox.test.ts | 8 +- .../nested-coordination-durability.test.ts | 191 +++++- tests/kernel/spawn-forest.test.ts | 251 +++++++ tests/kernel/supervise-convenience.test.ts | 51 +- .../supervise-full-profile-bridge.test.ts | 363 +++++++++- tests/kernel/supervise.test.ts | 225 ++++++- 33 files changed, 2829 insertions(+), 494 deletions(-) create mode 100644 src/runtime/supervise/tree-key.ts create mode 100644 tests/candidate-execution-export-surface.test.ts create mode 100644 tests/kernel/spawn-forest.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b7b60c69..38f5fb06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.110.0 + +- Run every supervisor, including the root, from one complete `AgentProfile`, preserve exact profile/task/candidate identity through recursive delegation, and reject execution paths that would silently drop profile fields. +- Expose node-scoped product tools, product authorization for exact spawns and continuations, awaited replay-safe coordination observation, structured worker traces, trace-derived failure guidance, and caller cancellation across the complete recursive run. +- Make durable run and assignment identity stable across restart while retaining exact materialization, accounting, delivery, and settlement evidence for each node. +- Add live root-manager steering, trusted post-authorization manager/leaf classification, per-assignment completion checks, a cold recursive forest reader, and public exact-profile candidate conversion helpers. +- Align Runtime with Eval 0.135.4, Interface 0.37.0, Knowledge 7.0.0, and Materialize 0.9.3 so public-source provenance and exact knowledge claims use one package cohort. + ## 0.109.2 - Align Runtime with Eval 0.135.2 and Knowledge 6.1.11 so every improvement path uses the corrected paired promotion decisions. diff --git a/docs/api/candidate-execution.md b/docs/api/candidate-execution.md index ff931393..69203b88 100644 --- a/docs/api/candidate-execution.md +++ b/docs/api/candidate-execution.md @@ -272,6 +272,18 @@ Re-exports [assertCandidateProfileBinding](index.md#assertcandidateprofilebindin *** +### freezeGenericAgentCandidateProfile + +Re-exports [freezeGenericAgentCandidateProfile](index.md#freezegenericagentcandidateprofile) + +*** + +### omitUndefinedObjectFields + +Re-exports [omitUndefinedObjectFields](index.md#omitundefinedobjectfields) + +*** + ### parseExactAgentProfile Re-exports [parseExactAgentProfile](index.md#parseexactagentprofile) @@ -284,6 +296,12 @@ Re-exports [parseExactAgentProfileDiff](index.md#parseexactagentprofilediff) *** +### parseExactCandidateProfile + +Re-exports [parseExactCandidateProfile](index.md#parseexactcandidateprofile) + +*** + ### AgentCandidateModelGrantActivateInput Re-exports [AgentCandidateModelGrantActivateInput](index.md#agentcandidatemodelgrantactivateinput) diff --git a/docs/api/index.md b/docs/api/index.md index 88b6c2e0..1def30ad 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -7981,7 +7981,7 @@ own agent (mastra/agno/raw HTTP/anything) is first-class by implementing this in ##### runtime -> `readonly` **runtime**: [`Runtime`](runtime.md#runtime-2) +> `readonly` **runtime**: [`Runtime`](runtime.md#runtime-4) Stable runtime tag for traces + the equal-k exemption check. @@ -8020,7 +8020,7 @@ the terminal artifact is read from `resultArtifact()` after the stream drains. ##### deliver()? -> `optional` **deliver**(`msg`): `void` +> `optional` **deliver**(`msg`): `boolean` \| `void` Optional inbox: receive an out-of-band message from the driver mid-run (the `send`/`steer_agent` verb). A streaming executor drains pending messages between turns and folds them into the next @@ -8036,7 +8036,7 @@ executor's to ignore. ###### Returns -`void` +`boolean` \| `void` ##### progress()? @@ -8219,7 +8219,7 @@ Register a factory for a named runtime. Throws on a duplicate name (fail loud). ###### runtime -[`Runtime`](runtime.md#runtime-2) +[`Runtime`](runtime.md#runtime-4) ###### factory @@ -8627,7 +8627,7 @@ live `RootHandle` (the Q2 substrate the chat/pi-viz client later consumes). ###### h -[`RootHandle`](runtime.md#roothandle)\<`Out`\> +[`RootHandle`](runtime.md#roothandle-1)\<`Out`\> ###### Returns @@ -12491,6 +12491,24 @@ Materializes a verified candidate into one immutable evaluator-owned execution p *** +### freezeGenericAgentCandidateProfile() + +> **freezeGenericAgentCandidateProfile**(`input`): `AgentCandidateProfile` + +Convert only behavior-preserving generic profile fields into the closed candidate contract. + +#### Parameters + +##### input + +`AgentProfile` + +#### Returns + +`AgentCandidateProfile` + +*** + ### assertCandidateProfileBinding() > **assertCandidateProfileBinding**(`measuredInput`, `bundled`): `void` @@ -12583,6 +12601,24 @@ Apply one exact diff and reject any value that cannot be preserved canonically. *** +### parseExactCandidateProfile() + +> **parseExactCandidateProfile**(`input`): `AgentCandidateProfile` + +Parse a candidate profile without silently discarding unsupported or non-canonical fields. + +#### Parameters + +##### input + +`unknown` + +#### Returns + +`AgentCandidateProfile` + +*** + ### agentCandidateProfileAsAgentProfile() > **agentCandidateProfileAsAgentProfile**(`candidate`): `AgentProfile` @@ -12601,6 +12637,28 @@ Convert the candidate profile contract into the portable interface profile it re *** +### omitUndefinedObjectFields() + +> **omitUndefinedObjectFields**(`value`, `path`): `unknown` + +Recursively remove undefined object fields while refusing undefined array entries. + +#### Parameters + +##### value + +`unknown` + +##### path + +`string` + +#### Returns + +`unknown` + +*** + ### createProtectedAgentCandidateModelPort() > **createProtectedAgentCandidateModelPort**(`options`): [`AgentCandidateModelPort`](#agentcandidatemodelport) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index c78992bf..9b9aefe4 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.109.2` and `@tangle-network/agent-eval@0.135.3` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.110.0` and `@tangle-network/agent-eval@0.135.4` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 404 exports. +Import from `@tangle-network/agent-runtime` — 407 exports. | Symbol | Kind | Summary | |---|---|---| @@ -64,6 +64,7 @@ Import from `@tangle-network/agent-runtime` — 404 exports. | `exportEvalRuns` | function | Ship self-improvement eval-run events to Tangle Intelligence. Unlike the | | `findingLines` | function | Render findings as the ranked-evidence block every build prompt ends with. | | `formatSupervisedKnowledgeTask` | function | Format the supervisor task with the KB root, readiness requirements, current findings, and metadata. | +| `freezeGenericAgentCandidateProfile` | function | Convert only behavior-preserving generic profile fields into the closed candidate contract. | | `getModels` | function | Fetch the model catalog from the router's `/v1/models`. Throws on a non-2xx | | `improve` | function | Optimize one exact profile surface with a complete method. | | `isDelegatedLoopMode` | function | Type guard — returns true when `value` is a valid `DelegatedLoopMode` string. | @@ -80,8 +81,10 @@ Import from `@tangle-network/agent-runtime` — 404 exports. | `notifyRuntimeHookEvent` | function | Fire `hooks.onEvent`, swallowing sync throws and surfacing async failures to `onError`. | | `officialGepa` | function | Build a complete method backed by GEPA's official Optimize Anything API. | | `officialSkillOpt` | function | Build a complete method backed by Microsoft's official SkillOpt trainer. | +| `omitUndefinedObjectFields` | function | Recursively remove undefined object fields while refusing undefined array entries. | | `parseExactAgentProfile` | function | Parse a complete profile without silently discarding unsupported fields. | | `parseExactAgentProfileDiff` | function | Parse a profile diff without silently discarding unsupported fields. | +| `parseExactCandidateProfile` | function | Parse a candidate profile without silently discarding unsupported or non-canonical fields. | | `parseLoopRunnerArgv` | function | Parse `--mode X --config Y` from an argv tail (`process.argv.slice(2)`). | | `parseRolloutPolicy` | function | Parse a serialized policy surface. Returns `undefined` for non-strings, | | `persistCandidateOutputArtifact` | function | Persist evaluator evidence, read it back, and bind the returned locator to the exact bytes. | @@ -502,7 +505,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 631 exports. +Import from `@tangle-network/agent-runtime/kernel` — 643 exports. | Symbol | Kind | Summary | |---|---|---| @@ -588,6 +591,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 631 exports. | `isWaitOutcome` | function | Narrow a settlement's `out` to a wait outcome — a wait settles on the SAME cursor as workers, | | `jjWorkspace` | function | A jj-backed `Workspace` (Jujutsu, colocated with git for the durable remote). | | `leaderboard` | function | Aggregate a fleet of records into the ranked, multi-axis report. Pure — no IO, deterministic. | +| `loadSpawnForest` | function | Load every journal tree owned by one recursive supervision run and flatten its nodes/events. | | `localSandboxClient` | function | A same-host `SandboxClient` adapter with no process isolation. Local MCP is | | `localShell` | function | Host-process `Shell`: run a command via `execFile`, resolving `{ stdout, stderr, code }` (never throws on non-zero exit). | | `loopCampaignDispatch` | function | Adapter for plain `runCampaign` scenarios. This is the Runtime-side pair for | @@ -731,6 +735,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 631 exports. | `AnalystFindingEvent` | interface | A trace-analyst result re-entered as a message on the bus (the `finding` event kind). | | `AuthorizedDownMessage` | interface | Product-authorized continuation bytes. Returning a narrowed instruction replaces the proposed | | `AuthorizedSpawn` | interface | The product-authorized result for one complete spawn request. Attribution is never accepted | +| `AuthorizedSpawnContext` | interface | Exact trusted context after a manager-authored spawn has passed product authorization. | | `BenchmarkCell` | interface | One strategy's outcome on one task — the per-task cell an optimizer consumes. | | `BenchmarkReport` | interface | Benchmark output: per-strategy means plus the full per-task × per-strategy losses table an optimizer mines. | | `BridgeSeam` | interface | cli-bridge seam. A local OpenAI-compatible bridge that fronts harness CLIs | @@ -769,6 +774,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 631 exports. | `DownMessageAuthorizationInput` | interface | Detached continuation bytes and exact worker identity presented to product authorization before | | `DownMessageDeliveryAttempt` | interface | A durable marker written after authorization and immediately before Runtime calls `Scope.send`. | | `DownMessageEvent` | interface | A parent→child delivery result (the down-leg): recorded for observability, never pulled back by | +| `DriveHarness` | interface | How to run an external harness as the DRIVER, with the coordination verbs mounted — the substrate | | `DumbDriverOptions` | interface | Options for {@link dumbDriver}. | | `EqualKArm` | interface | One arm of an equal-k comparison — a labeled trajectory (a `TrajectoryReport` is one arm's whole | | `EqualKOnCostOptions` | interface | `equalKOnCost(arms, { tolerance? })` — assert arms are comparable at EQUAL conserved COST | @@ -846,7 +852,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 631 exports. | `ResultBlobStore` | interface | Content-addressed result blobs (the `outRef` → artifact map) backing the replay | | `ResumedKeyState` | interface | What the journal proves about one keyed assignment at resume time. | | `ResumedWork` | interface | The committed work a resumed run inherits from its journal. `settled` is the replayed | -| `RootHandle` | interface | Live root handle — the substrate a chat/pi-viz client attaches to (Q2). `signal` | +| `RootHandle` | interface | Live root handle — a chat/pi-viz client uses it to inspect and control one root run. | | `RouterSeam` | interface | Router/inline connection seam. A direct OpenAI-compatible Router endpoint — | | `RouterToolCall` | interface | A tool-call the model emitted (provider-neutral; mirrors the runtime's ToolCallRequest). | | `RouterToolsSeam` | interface | Router seam WITH tool use — the tool-using router backend. Same direct | @@ -875,8 +881,15 @@ Import from `@tangle-network/agent-runtime/kernel` — 631 exports. | `ShapeContext` | interface | The construction context a `LoopShape` factory receives. Carries the persona's resolved | | `ShapeRegistry` | interface | The open shape registry — the extension point that makes a new loop-shape ONE file + one | | `ShotPersona` | interface | A role for one shot — multi-agent loops (researcher + engineer, a panel of k | +| `SpawnForest` | interface | Complete cold-readable view of one recursive supervision run. | +| `SpawnForestEvent` | interface | One event with the journal tree that establishes its cursor namespace. | +| `SpawnForestInDoubtNode` | interface | A spawned worker with no terminal record in a cold snapshot. Resume treats the same state as | +| `SpawnForestMissingTree` | interface | A driver spawn whose owned journal tree was never begun before the process stopped. | +| `SpawnForestNode` | interface | One flattened node with the journal tree that owns its records. | +| `SpawnForestTree` | interface | One journal tree in a recursively loaded supervision forest. | | `SpawnJournal` | interface | The spawn-tree event source (mirrors `ConversationJournal`'s begin/append/load shape). | | `Spend` | interface | Conserved spend, reconciled from the normalized `UsageEvent` stream. Tokens and usd | +| `SteerableRootHandle` | interface | A Runtime-minted root handle that can deliver raw steering or answers to a live manager inbox. | | `SteerableSandboxSession` | interface | What the steerable session exposes to its executor: the usage stream plus the live reads. | | `SteerContext` | interface | How a combinator's `act` consumes findings to steer — the SINGLE firewalled steer surface a | | `StrategyArtifacts` | interface | Artifact lifecycle a strategy may manage itself — open/close ONLY. Raw `call`/`score` | @@ -925,9 +938,10 @@ Import from `@tangle-network/agent-runtime/kernel` — 631 exports. | `CoordinationOwnerId` | type | Stable identity of the supervisor that owns one coordination stream. High-level supervision | | `DefinePersona` | type | Builds a frozen `Persona`, failing loud on the executors-supplied invariant (neither a | | `Deliverable` | type | How a typed deliverable `Out` is materialized from a finished turn. | +| `DeliverableResolutionInput` | type | Exact trusted context for selecting one backend-derived leaf's completion check. | | `DispatchStopReason` | type | Why the dispatcher stopped admitting work. `drained` = the queue ran dry (the ordinary end); | | `DownMessageDeliveryOutcome` | type | The exact result of one parent→child delivery attempt. | -| `DriveHarness` | type | How to run an external harness as the DRIVER, with the coordination verbs mounted — the substrate | +| `DriveHarnessOwnerContext` | type | Trusted manager identity available before its external harness starts. A product uses this to | | `Environment` | type | A checkable task domain — implement these 5 hooks and the suite does the rest. The | | `EqualKOnCost` | type | `equalKOnCost(arms, opts)` — the cross-arm equal-compute check on conserved cost. | | `ExecutionBindingReceipt` | type | One attempt's immutable link from a stable materialization plan to its actual transport. | @@ -953,6 +967,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 631 exports. | `ProfileKeyOf` | type | The profile (matrix row) a record belongs to — default `harness·model` from the record's profile cell, | | `ProfileMaterializationReceipt` | type | What the kernel can prove about one node's actual execution plan. | | `RenderCorpusToInstructions` | type | `renderCorpusToInstructions(opts)` — the flywheel read-back projection. Async (queries the | +| `ResolveDriveHarness` | type | Resolve an external harness for one exact Runtime-owned manager identity. | | `ResolveSupervisorTools` | type | Product policy for the tools one exact supervisor node may call. Resolved once per node. | | `Restart` | type | OTP child-spec restart class. | | `RootMaterialization` | type | Trusted root composition evidence. Generic `Agent.act` roots omit this and remain unknown. | @@ -1135,7 +1150,7 @@ Import from `@tangle-network/agent-runtime/primeintellect` — 30 exports. ### Candidate execution — immutable prepare, run, grade, and receipt -Import from `@tangle-network/agent-runtime/candidate-execution` — 105 exports. +Import from `@tangle-network/agent-runtime/candidate-execution` — 108 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1152,8 +1167,11 @@ Import from `@tangle-network/agent-runtime/candidate-execution` — 105 exports. | `disposePreparedAgentCandidateExecution` | function | Revoke reservations held by a prepared candidate that will not be executed. | | `exactProcessProviderAsCandidateExecutor` | function | Adapt one neutral exact-process provider to Runtime's trusted candidate boundary. | | `executePreparedAgentCandidate` | function | Executes and finalizes one durably claimed candidate without exposing an unproven result. | +| `freezeGenericAgentCandidateProfile` | function | Convert only behavior-preserving generic profile fields into the closed candidate contract. | +| `omitUndefinedObjectFields` | function | Recursively remove undefined object fields while refusing undefined array entries. | | `parseExactAgentProfile` | function | Parse a complete profile without silently discarding unsupported fields. | | `parseExactAgentProfileDiff` | function | Parse a profile diff without silently discarding unsupported fields. | +| `parseExactCandidateProfile` | function | Parse a candidate profile without silently discarding unsupported or non-canonical fields. | | `persistCandidateOutputArtifact` | function | Persist evaluator evidence, read it back, and bind the returned locator to the exact bytes. | | `prepareAgentCandidateExecution` | function | Materializes a verified candidate into one immutable evaluator-owned execution plan. | | `recoverExpiredAgentCandidateExecution` | function | Close an expired crashed attempt from persisted non-secret handles, then record failure. | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 0f118560..e6a71382 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -667,6 +667,273 @@ FS-backed `CoordinationLog`: append-only JSONL, fsynced per record. ## Interfaces +### SpawnForestTree + +One journal tree in a recursively loaded supervision forest. + +#### Properties + +##### root + +> `readonly` **root**: `string` + +##### ownerNodeId? + +> `readonly` `optional` **ownerNodeId?**: `string` + +Driver node that owns this tree; absent for the requested root tree. + +##### parentTreeRoot? + +> `readonly` `optional` **parentTreeRoot?**: `string` + +Journal tree containing `ownerNodeId`; absent for the requested root tree. + +##### events + +> `readonly` **events**: readonly [`SpawnEvent`](#spawnevent)[] + +##### view + +> `readonly` **view**: [`TreeView`](#treeview) + +*** + +### SpawnForestEvent + +One event with the journal tree that establishes its cursor namespace. + +#### Properties + +##### treeRoot + +> `readonly` **treeRoot**: `string` + +##### event + +> `readonly` **event**: [`SpawnEvent`](#spawnevent) + +*** + +### SpawnForestNode + +One flattened node with the journal tree that owns its records. + +#### Extends + +- [`NodeSnapshot`](#nodesnapshot) + +#### Properties + +##### treeRoot + +> `readonly` **treeRoot**: `string` + +##### id + +> `readonly` **id**: `string` + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`id`](#id-17) + +##### parent? + +> `readonly` `optional` **parent?**: `string` + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`parent`](#parent-4) + +##### label + +> `readonly` **label**: `string` + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`label`](#label-17) + +##### status + +> `readonly` **status**: [`NodeStatus`](#nodestatus) + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`status`](#status-9) + +##### runtime + +> `readonly` **runtime**: [`Runtime`](#runtime-4) + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`runtime`](#runtime-5) + +##### budget + +> `readonly` **budget**: [`Budget`](index.md#budget-4) + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`budget`](#budget-17) + +##### assignmentId? + +> `readonly` `optional` **assignmentId?**: `string` + +Manager-scoped assignment identity, including deterministic ids for unkeyed siblings. + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`assignmentId`](#assignmentid-6) + +##### identity? + +> `readonly` `optional` **identity?**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`identity`](#identity-5) + +##### materialization? + +> `readonly` `optional` **materialization?**: [`ProfileMaterializationReceipt`](#profilematerializationreceipt) + +Kernel-owned execution evidence. `unknown` is distinct from a known zero/empty plan. + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`materialization`](#materialization-2) + +##### executionBindings? + +> `readonly` `optional` **executionBindings?**: readonly [`ExecutionBindingReceipt`](#executionbindingreceipt)[] + +Immutable attempt bindings, oldest first. A retried/resumed node may have more than one. + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`executionBindings`](#executionbindings-2) + +##### settledAt? + +> `readonly` `optional` **settledAt?**: `number` + +Epoch ms of the terminal journal record; absent while live or when legacy evidence lacks it. + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`settledAt`](#settledat-1) + +##### spent + +> `readonly` **spent**: [`Spend`](index.md#spend) + +Conserved spend so far for this node. + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`spent`](#spent-2) + +##### outRef? + +> `readonly` `optional` **outRef?**: `string` + +`outRef` once the node is `done` (the replay/result pointer). + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`outRef`](#outref-5) + +##### trace? + +> `readonly` `optional` **trace?**: [`WorkerTraceEvidence`](index.md#workertraceevidence) + +Present on terminal executor nodes; legacy records carry an explicit unavailable reason. + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`trace`](#trace-3) + +*** + +### SpawnForestInDoubtNode + +A spawned worker with no terminal record in a cold snapshot. Resume treats the same state as +in-doubt and conservatively retains its reservation. Root nodes and armed waits are excluded. + +#### Properties + +##### treeRoot + +> `readonly` **treeRoot**: `string` + +##### nodeId + +> `readonly` **nodeId**: `string` + +##### label + +> `readonly` **label**: `string` + +##### runtime + +> `readonly` **runtime**: [`Runtime`](#runtime-4) + +*** + +### SpawnForestMissingTree + +A driver spawn whose owned journal tree was never begun before the process stopped. + +#### Properties + +##### parentTreeRoot + +> `readonly` **parentTreeRoot**: `string` + +##### ownerNodeId + +> `readonly` **ownerNodeId**: `string` + +##### root + +> `readonly` **root**: `string` + +*** + +### SpawnForest + +Complete cold-readable view of one recursive supervision run. + +#### Properties + +##### root + +> `readonly` **root**: `string` + +##### trees + +> `readonly` **trees**: readonly [`SpawnForestTree`](#spawnforesttree)[] + +##### nodes + +> `readonly` **nodes**: readonly [`SpawnForestNode`](#spawnforestnode)[] + +##### events + +> `readonly` **events**: readonly [`SpawnForestEvent`](#spawnforestevent)[] + +##### inDoubt + +> `readonly` **inDoubt**: readonly [`SpawnForestInDoubtNode`](#spawnforestindoubtnode)[] + +##### missingTrees + +> `readonly` **missingTrees**: readonly [`SpawnForestMissingTree`](#spawnforestmissingtree)[] + +*** + ### AnalystFindingEvent A trace-analyst result re-entered as a message on the bus (the `finding` event kind). @@ -3656,7 +3923,7 @@ OTP intensity breaker bounds, forwarded to the supervisor verbatim. ##### handle? -> `readonly` `optional` **handle?**: [`RootHandle`](#roothandle)\<[`Outcome`](#outcome-2)\<`D`\>\> +> `readonly` `optional` **handle?**: [`RootHandle`](#roothandle-1)\<[`Outcome`](#outcome-2)\<`D`\>\> A live root handle to attach (view/signal/abort) before the run starts. @@ -9609,6 +9876,13 @@ How the settled-worker ledger becomes the run's output. Default `bestDelivered` the delivered-only invariant (`runFinalizer`): whatever the finalizer, an undelivered or invalid child's output stays unreachable. +##### inbox? + +> `readonly` `optional` **inbox?**: [`Inbox`](#inbox-1) + +Optional shared manager inbox used by a wrapper that must accept messages before async node +setup finishes. Ordinary callers omit it and the driver owns a fresh inbox. + *** ### PriorCoordination @@ -11161,7 +11435,7 @@ Generic environment provider executor config. External packages implement ##### runtime? -> `optional` **runtime?**: [`Runtime`](#runtime-2) +> `optional` **runtime?**: [`Runtime`](#runtime-4) **`Experimental`** @@ -11434,7 +11708,7 @@ Drive the worker to settlement. `signal` is the spawn-scoped abort handed to `ex ##### inbox -> `readonly` **inbox**: [`Inbox`](#inbox) +> `readonly` **inbox**: [`Inbox`](#inbox-1) ##### taskToPrompt @@ -11979,6 +12253,13 @@ Idle time that counts as stalled, passed through to the live progress read. Omit The conserved compute pool for the whole run. +##### rootHandle? + +> `readonly` `optional` **rootHandle?**: [`RootHandle`](#roothandle-1)\<`unknown`\> + +Caller-created live handle for observing, steering, or cancelling this root manager. Runtime +attaches it before execution and detaches it after the join barrier. + ##### signal? > `readonly` `optional` **signal?**: `AbortSignal` @@ -12007,6 +12288,24 @@ The completion oracle for backend-derived workers (settled ⟺ delivered). Stron without it the supervisor trusts a worker's self-report — exactly the "ran but didn't deliver" failure mode of a static orchestrator. +##### resolveDeliverable? + +> `readonly` `optional` **resolveDeliverable?**: (`input`) => [`DeliverableSpec`](#deliverablespec)\<`unknown`\> \| `undefined` + +Resolve the completion check for one exact authorized backend-derived leaf. The callback runs +after spawn authorization and driver classification, receives a detached immutable context, +and may return `undefined` to use the run-wide `deliverable`. Driver profiles never call it. + +###### Parameters + +###### input + +[`AuthorizedSpawnContext`](#authorizedspawncontext) + +###### Returns + +[`DeliverableSpec`](#deliverablespec)\<`unknown`\> \| `undefined` + ##### makeWorkerAgent? > `readonly` `optional` **makeWorkerAgent?**: [`MakeWorkerAgent`](#makeworkeragent) @@ -12118,23 +12417,15 @@ authorized task. The exact worker identity and detached bytes are recorded befor > `readonly` `optional` **isDriverProfile?**: (`input`) => `boolean` Decide whether an authorized child becomes another supervisor. By default only - `metadata.role === 'driver'` does; products may bind this to their own authority record. + `metadata.role === 'driver'` does. Products receive the same frozen post-authorization + context as `resolveDeliverable`, so trusted execution/assignment authority can override + model-authored metadata without a side channel. ###### Parameters ###### input -###### profile - -`AgentProfile` - -###### parent - -`AgentProfile` - -###### depth - -`number` +[`AuthorizedSpawnContext`](#authorizedspawncontext) ###### Returns @@ -12160,12 +12451,20 @@ Inject the supervisor brain directly (tests / advanced). Run an external-harness supervisor explicitly. Required for a remote sandbox; optional as a caller-owned override for a local bridge. +##### resolveDriveHarness? + +> `readonly` `optional` **resolveDriveHarness?**: [`ResolveDriveHarness`](#resolvedriveharness-1) + +Resolve one custom external-harness session per trusted manager identity. Use this instead of +`driveHarness` when recursive managers must be independently steerable. + ##### driveHarnessMaterialization? > `readonly` `optional` **driveHarnessMaterialization?**: [`ProfileMaterializationContract`](agent.md#profilematerializationcontract) -Required with a custom `driveHarness`: declares which complete AgentProfile axes that path -really applies. Built-in bridge driving supplies its own full-profile contract. +Required with a custom `driveHarness` or `resolveDriveHarness`: declares which complete +AgentProfile axes that path really applies. Built-in bridge driving supplies its own +full-profile contract. ##### resolveSupervisorTools? @@ -12421,6 +12720,58 @@ from the manager itself; it enters only through this trusted callback. *** +### AuthorizedSpawnContext + +Exact trusted context after a manager-authored spawn has passed product authorization. + +#### Properties + +##### profile + +> `readonly` **profile**: `AgentProfile` + +##### parent + +> `readonly` **parent**: `AgentProfile` + +##### parentIdentity + +> `readonly` **parentIdentity**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +##### execution + +> `readonly` **execution**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +##### parentNodeId + +> `readonly` **parentNodeId**: `string` + +##### assignmentId + +> `readonly` **assignmentId**: `string` + +##### task + +> `readonly` **task**: `unknown` + +##### budget + +> `readonly` **budget**: [`Budget`](index.md#budget-4) + +##### label + +> `readonly` **label**: `string` + +##### key? + +> `readonly` `optional` **key?**: `string` + +##### depth + +> `readonly` **depth**: `number` + +*** + ### SupervisorNodeContext Trusted run/node identity Runtime binds to one manager. Model-authored tool arguments cannot @@ -12529,6 +12880,72 @@ One product-owned tool. It reuses the canonical MCP descriptor fields while Runt *** +### DriveHarness() + +How to run an external harness as the DRIVER, with the coordination verbs mounted — the substrate + seam the caller supplies (mirrors `makeWorkerAgent` for spawned children). It runs `profile` on + `task` in its backend (remote sandbox or local CLI bridge) with `coordinationMcpUrl` mounted as an MCP server, + so the harness calls spawn_agent / await_event / stop as native tools over the live scope. + +> **DriveHarness**(`args`): `Promise`\<`void`\> + +How to run an external harness as the DRIVER, with the coordination verbs mounted — the substrate + seam the caller supplies (mirrors `makeWorkerAgent` for spawned children). It runs `profile` on + `task` in its backend (remote sandbox or local CLI bridge) with `coordinationMcpUrl` mounted as an MCP server, + so the harness calls spawn_agent / await_event / stop as native tools over the live scope. + +#### Parameters + +##### args + +###### profile + +`AgentProfile` + +###### task + +`unknown` + +###### scope + +[`Scope`](index.md#scope)\<`unknown`\> + +###### coordinationMcpUrl + +`string` + +###### coordinationTools + +readonly `Omit`\<[`McpToolDescriptor`](mcp.md#mcptooldescriptor), `"handler"`\>[] + +Data-only product tool surface mounted on the coordination MCP. Runtime-owned drivers include + this in their materialization evidence without persisting executable handlers. + +#### Returns + +`Promise`\<`void`\> + +#### Methods + +##### deliver()? + +> `optional` **deliver**(`message`): `boolean` + +Optional live inbox for the manager session this adapter currently drives. Return `false` +when no executor inbox is active instead of claiming a message was delivered. + +###### Parameters + +###### message + +`unknown` + +###### Returns + +`boolean` + +*** + ### SupervisorAgentDeps #### Properties @@ -12960,6 +13377,24 @@ unordered collection. `scope.next()` delivers strictly in recorded `seq` order. `Promise`\<`Out`\> +##### deliver()? + +> `optional` **deliver**(`msg`): `boolean` \| `void` + +Optional manager inbox. A parent or attached `RootHandle` uses this to deliver the same raw +down-message accepted by executor inboxes. Return `false` when the manager has no live receive +path; returning `true` means the message was accepted for the current manager session. + +###### Parameters + +###### msg + +`unknown` + +###### Returns + +`boolean` \| `void` + *** ### ExecutorAccounting @@ -13431,6 +13866,10 @@ The rehydrated settlement; absent exactly when `state` is `'in-doubt'`. ### NodeSnapshot +#### Extended by + +- [`SpawnForestNode`](#spawnforestnode) + #### Properties ##### id @@ -13451,7 +13890,7 @@ The rehydrated settlement; absent exactly when `state` is `'in-doubt'`. ##### runtime -> `readonly` **runtime**: [`Runtime`](#runtime-2) +> `readonly` **runtime**: [`Runtime`](#runtime-4) ##### budget @@ -13752,8 +14191,9 @@ Lifecycle stream sink, threaded into the root `Scope` so every `spawn`/settle em ### RootHandle -Live root handle — the substrate a chat/pi-viz client attaches to (Q2). `signal` - delivers an out-of-band message to the running root; `view()` materializes the tree. +Live root handle — the substrate a chat/pi-viz client attaches to (Q2). `deliver` + sends a raw steer/answer to a manager inbox, `signal` controls the run, and `view()` + materializes the tree. #### Type Parameters @@ -13780,6 +14220,23 @@ Phantom: binds the handle to the supervised run's output type. Type-only — nev [`TreeView`](#treeview) +##### deliver() + +> **deliver**(`msg`): `boolean` + +Deliver a raw down-message to the live root manager. Returns `false` when that manager has no +inbox. Before binding and after completion this fails loud like the other handle methods. + +###### Parameters + +###### msg + +`unknown` + +###### Returns + +`boolean` + ##### signal() > **signal**(`msg`): `void` @@ -16685,7 +17142,7 @@ judge/verdict/score scheme is rejected. Fail loud — a tainted finding aborts. ##### root -[`NodeId`](#nodeid-3) +[`NodeId`](#nodeid-4) ##### options? @@ -17059,6 +17516,14 @@ Evaluated from the progress feed, never from the budget. Pure and synchronous: i *** +### DeliverableResolutionInput + +> **DeliverableResolutionInput** = [`AuthorizedSpawnContext`](#authorizedspawncontext) + +Exact trusted context for selecting one backend-derived leaf's completion check. + +*** + ### SupervisorProfile > **SupervisorProfile** = `AgentProfile` @@ -17122,45 +17587,30 @@ Context-aware observer used internally to bind product transactions to the actua *** -### DriveHarness +### DriveHarnessOwnerContext -> **DriveHarness** = (`args`) => `Promise`\<`void`\> +> **DriveHarnessOwnerContext** = `Omit`\<[`SupervisorNodeContext`](#supervisornodecontext), `"nodeId"`\> -How to run an external harness as the DRIVER, with the coordination verbs mounted — the substrate - seam the caller supplies (mirrors `makeWorkerAgent` for spawned children). It runs `profile` on - `task` in its backend (remote sandbox or local CLI bridge) with `coordinationMcpUrl` mounted as an MCP server, - so the harness calls spawn_agent / await_event / stop as native tools over the live scope. +Trusted manager identity available before its external harness starts. A product uses this to +return one independently steerable harness session per recursive manager. -#### Parameters - -##### args - -###### profile - -[`SupervisorProfile`](#supervisorprofile) - -###### task - -`unknown` - -###### scope +*** -[`Scope`](index.md#scope)\<`unknown`\> +### ResolveDriveHarness -###### coordinationMcpUrl +> **ResolveDriveHarness** = (`context`) => [`DriveHarness`](#driveharness-1) -`string` +Resolve an external harness for one exact Runtime-owned manager identity. -###### coordinationTools +#### Parameters -`ReadonlyArray`\<`Omit`\<[`McpToolDescriptor`](mcp.md#mcptooldescriptor), `"handler"`\>\> +##### context -Data-only product tool surface mounted on the coordination MCP. Runtime-owned drivers include - this in their materialization evidence without persisting executable handlers. +[`DriveHarnessOwnerContext`](#driveharnessownercontext) #### Returns -`Promise`\<`void`\> +[`DriveHarness`](#driveharness-1) *** @@ -17201,7 +17651,7 @@ Why exact materialization evidence is unavailable for a node. ### ProfileMaterializationReceipt -> **ProfileMaterializationReceipt** = \{ `status`: `"known"`; `authoredProfileDigest`: `Sha256Digest`; `effectiveProfileDigest`: `Sha256Digest`; `materializationPlanDigest`: `Sha256Digest`; `platformAttachmentsDigest?`: `Sha256Digest`; `runtime`: [`Runtime`](#runtime-2); `backend`: `string`; `model`: [`MaterializedModelIdentity`](#materializedmodelidentity); `execution`: [`MaterializedExecutionIdentity`](#materializedexecutionidentity); `materializer`: `string`; \} \| \{ `status`: `"unknown"`; `authoredProfileDigest?`: `Sha256Digest`; `runtime`: [`Runtime`](#runtime-2); `reason`: [`UnknownMaterializationReason`](#unknownmaterializationreason); \} +> **ProfileMaterializationReceipt** = \{ `status`: `"known"`; `authoredProfileDigest`: `Sha256Digest`; `effectiveProfileDigest`: `Sha256Digest`; `materializationPlanDigest`: `Sha256Digest`; `platformAttachmentsDigest?`: `Sha256Digest`; `runtime`: [`Runtime`](#runtime-4); `backend`: `string`; `model`: [`MaterializedModelIdentity`](#materializedmodelidentity); `execution`: [`MaterializedExecutionIdentity`](#materializedexecutionidentity); `materializer`: `string`; \} \| \{ `status`: `"unknown"`; `authoredProfileDigest?`: `Sha256Digest`; `runtime`: [`Runtime`](#runtime-4); `reason`: [`UnknownMaterializationReason`](#unknownmaterializationreason); \} What the kernel can prove about one node's actual execution plan. @@ -17217,7 +17667,7 @@ One attempt's immutable link from a stable materialization plan to its actual tr ### RootMaterialization -> **RootMaterialization** = \{ `runtime`: [`Runtime`](#runtime-2); `declaration`: [`ExecutorMaterialization`](#executormaterialization); `binding`: `Omit`\<[`ExecutorExecutionBinding`](#executorexecutionbinding), `"attemptId"`\>; \} \| \{ `runtime`: [`Runtime`](#runtime-2); `declaration`: `"deferred"`; `authoredProfile`: `AgentProfile`; \} +> **RootMaterialization** = \{ `runtime`: [`Runtime`](#runtime-4); `declaration`: [`ExecutorMaterialization`](#executormaterialization); `binding`: `Omit`\<[`ExecutorExecutionBinding`](#executorexecutionbinding), `"attemptId"`\>; \} \| \{ `runtime`: [`Runtime`](#runtime-4); `declaration`: `"deferred"`; `authoredProfile`: `AgentProfile`; \} Trusted root composition evidence. Generic `Agent.act` roots omit this and remain unknown. @@ -17225,17 +17675,17 @@ Trusted root composition evidence. Generic `Agent.act` roots omit this and remai ##### Type Literal -\{ `runtime`: [`Runtime`](#runtime-2); `declaration`: [`ExecutorMaterialization`](#executormaterialization); `binding`: `Omit`\<[`ExecutorExecutionBinding`](#executorexecutionbinding), `"attemptId"`\>; \} +\{ `runtime`: [`Runtime`](#runtime-4); `declaration`: [`ExecutorMaterialization`](#executormaterialization); `binding`: `Omit`\<[`ExecutorExecutionBinding`](#executorexecutionbinding), `"attemptId"`\>; \} *** ##### Type Literal -\{ `runtime`: [`Runtime`](#runtime-2); `declaration`: `"deferred"`; `authoredProfile`: `AgentProfile`; \} +\{ `runtime`: [`Runtime`](#runtime-4); `declaration`: `"deferred"`; `authoredProfile`: `AgentProfile`; \} ###### runtime -> `readonly` **runtime**: [`Runtime`](#runtime-2) +> `readonly` **runtime**: [`Runtime`](#runtime-4) The runtime-owned external adapter will publish the exact declaration after its dynamic platform attachment (for example a coordination URL) exists and before paid work starts. @@ -17322,7 +17772,7 @@ Fail-closed spawn rejections: an exhausted pool, an exceeded recursion ceiling, ### SpawnPrior -> **SpawnPrior**\<`Out`\> = \{ `state`: `"completed"`; `settled`: [`Settled`](index.md#settled)\<`Out`\> & `object`; \} \| \{ `state`: `"retried"`; `priorId`: [`NodeId`](#nodeid-3); `reason`: `string`; \} \| \{ `state`: `"lost"`; `priorId`: [`NodeId`](#nodeid-3); \} +> **SpawnPrior**\<`Out`\> = \{ `state`: `"completed"`; `settled`: [`Settled`](index.md#settled)\<`Out`\> & `object`; \} \| \{ `state`: `"retried"`; `priorId`: [`NodeId`](#nodeid-4); `reason`: `string`; \} \| \{ `state`: `"lost"`; `priorId`: [`NodeId`](#nodeid-4); \} What a KEYED spawn resolved to when the key had a prior attempt. Absent on a fresh key (and on every unkeyed spawn). `'completed'` is the exactly-once path: NOTHING was spawned — the handle @@ -17345,7 +17795,7 @@ adoption state; none of the built-ins can today. ### SpawnEvent -> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-3); `parent?`: [`NodeId`](#nodeid-3); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-2); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-3); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-3); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-3); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-3); `reason`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-3); `parent?`: [`NodeId`](#nodeid-3); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-3); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-3); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} +> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-4); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-4); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-4); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-4); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-4); `reason`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-4); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-4); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO timestamp for human inspection only (NOT a replay input). @@ -17354,7 +17804,7 @@ Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO ##### Type Literal -\{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-3); `parent?`: [`NodeId`](#nodeid-3); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-2); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-4); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} ###### kind @@ -17362,11 +17812,11 @@ Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO ###### id -> **id**: [`NodeId`](#nodeid-3) +> **id**: [`NodeId`](#nodeid-4) ###### parent? -> `optional` **parent?**: [`NodeId`](#nodeid-3) +> `optional` **parent?**: [`NodeId`](#nodeid-4) ###### label @@ -17391,7 +17841,7 @@ Manager-scoped assignment identity used to join unkeyed and keyed work alike. ###### runtime -> **runtime**: [`Runtime`](#runtime-2) +> **runtime**: [`Runtime`](#runtime-4) ###### identity? @@ -17411,7 +17861,7 @@ Exact profile/task digests plus trusted candidate/campaign attribution when avai ##### Type Literal -\{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-3); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-4); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} ###### kind @@ -17422,7 +17872,7 @@ only by digest; descriptor fields are safe structural labels, never credential-b ###### id -> **id**: [`NodeId`](#nodeid-3) +> **id**: [`NodeId`](#nodeid-4) ###### binding @@ -17440,7 +17890,7 @@ only by digest; descriptor fields are safe structural labels, never credential-b ##### Type Literal -\{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-3); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-4); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} ###### kind @@ -17450,7 +17900,7 @@ Trusted runtime transformation from the authorized profile to actual wire bytes. ###### id -> **id**: [`NodeId`](#nodeid-3) +> **id**: [`NodeId`](#nodeid-4) ###### receipt @@ -17468,7 +17918,7 @@ Trusted runtime transformation from the authorized profile to actual wire bytes. ##### Type Literal -\{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-3); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-4); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} ###### kind @@ -17476,7 +17926,7 @@ Trusted runtime transformation from the authorized profile to actual wire bytes. ###### id -> **id**: [`NodeId`](#nodeid-3) +> **id**: [`NodeId`](#nodeid-4) ###### status @@ -17525,13 +17975,13 @@ Structured tool evidence. Optional only for journals written before trace captur ##### Type Literal -\{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-3); `reason`: `string`; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-4); `reason`: `string`; `seq`: `number`; `at`: `string`; \} *** ##### Type Literal -\{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-3); `parent?`: [`NodeId`](#nodeid-3); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} ###### kind @@ -17544,11 +17994,11 @@ A wait-state node was ARMED. Lives in the SPAWN-ORDINAL namespace (`seq` is the ###### id -> **id**: [`NodeId`](#nodeid-3) +> **id**: [`NodeId`](#nodeid-4) ###### parent? -> `optional` **parent?**: [`NodeId`](#nodeid-3) +> `optional` **parent?**: [`NodeId`](#nodeid-4) ###### label @@ -17574,7 +18024,7 @@ A wait-state node was ARMED. Lives in the SPAWN-ORDINAL namespace (`seq` is the ##### Type Literal -\{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-3); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-4); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} ###### kind @@ -17587,7 +18037,7 @@ A wait-state node SETTLED — the cursor-namespace twin of `settled`, kept disti ###### id -> **id**: [`NodeId`](#nodeid-3) +> **id**: [`NodeId`](#nodeid-4) ###### by @@ -17609,7 +18059,7 @@ A wait-state node SETTLED — the cursor-namespace twin of `settled`, kept disti ##### Type Literal -\{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-3); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-4); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} ###### kind @@ -17625,7 +18075,7 @@ A driver's OWN inference spend, journaled separately from spawned-child work — ###### id -> **id**: [`NodeId`](#nodeid-3) +> **id**: [`NodeId`](#nodeid-4) ###### spend @@ -18066,7 +18516,7 @@ an empty collection is a no-winner, not a winner wrapping `[]`. ### PI\_RUNTIME -> `const` **PI\_RUNTIME**: [`Runtime`](#runtime-2) = `'pi'` +> `const` **PI\_RUNTIME**: [`Runtime`](#runtime-4) = `'pi'` The runtime name `piExecutor` registers under. @@ -18152,6 +18602,39 @@ Stable content address shared by result and trace artifacts. *** +### loadSpawnForest() + +> **loadSpawnForest**(`journal`, `root`): `Promise`\<[`SpawnForest`](#spawnforest)\> + +Load every journal tree owned by one recursive supervision run and flatten its nodes/events. + +Nested driver tree keys are a Runtime implementation detail; callers should use this reader +instead of deriving or scanning keys themselves. The reader follows raw `spawned` records whose +runtime is `driver`, preserving each tree's independent cursor namespace on flattened events. +A driver whose subtree was never begun is reported in `missingTrees`; any spawned non-root node +without a terminal record is reported in `inDoubt`, matching resume's conservative lost-work +interpretation. + +This is a cold/quiescent reader, not a transaction across an actively mutating file. Every value +returned is a detached immutable snapshot, so later journal writes or caller mutation cannot +change the result already observed. + +#### Parameters + +##### journal + +[`SpawnJournal`](#spawnjournal) + +##### root + +`string` + +#### Returns + +`Promise`\<[`SpawnForest`](#spawnforest)\> + +*** + ### replaySpawnTree() > **replaySpawnTree**(`journal`, `blobs`, `root`): `Promise`\<[`Settled`](index.md#settled)\<`unknown`\>[]\> @@ -21617,13 +22100,13 @@ readonly [`FinalizerSettled`](#finalizersettled)[] ### createInbox() -> **createInbox**(): [`Inbox`](#inbox) +> **createInbox**(): [`Inbox`](#inbox-1) Create the worker-side inbox for the down-leg: the driver's `steer_agent` / `answer_question` messages queue here and the worker's loop drains them at step boundaries and before settle. #### Returns -[`Inbox`](#inbox) +[`Inbox`](#inbox-1) *** diff --git a/docs/api/runtime/environment-provider.md b/docs/api/runtime/environment-provider.md index 438fa85d..10144065 100644 --- a/docs/api/runtime/environment-provider.md +++ b/docs/api/runtime/environment-provider.md @@ -260,7 +260,7 @@ Options for running a provider as a supervise-mode executor. ##### runtime? -> `optional` **runtime?**: [`Runtime`](../runtime.md#runtime-2) +> `optional` **runtime?**: [`Runtime`](../runtime.md#runtime-4) **`Experimental`** diff --git a/docs/canonical-api.md b/docs/canonical-api.md index c7aed98e..b728916b 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,11 +4,11 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.109.2.** +> **Version 0.110.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> `agent-eval` must satisfy `>=0.135.3 <0.136.0`. +> `agent-eval` must satisfy `>=0.135.4 <0.136.0`. > `sandbox` must satisfy `>=0.15.0 <0.16.0`. -> Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.36.0 <0.37.0`. +> Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.37.0 <0.38.0`. > > **`./kernel` is the execution kernel**: `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/kernel` lives there — the recursive atom (`Scope`/`Supervisor`), the executor registry, budget conservation, the finalizer seam, analyst wiring, and the round-synchronous loop. > @@ -55,13 +55,17 @@ Skills remain separate resources that the runtime can invoke; do not concatenate `supervise(profile, task, { backend, ... })` keeps that contract at execution time. It validates and freezes each authored child as a canonical `AgentProfile`, applies the shared profile-security policy and optional product authorization before reserving compute, and sends the exact authorized profile to the selected backend. -A child with `metadata.role: 'driver'` becomes another supervisor over the same budget and may author its own children; every other child is a leaf. +By default, a child with `metadata.role: 'driver'` becomes another supervisor over the same budget and may author its own children; every other child is a leaf. +Products that keep authority outside model-authored metadata pass `isDriverProfile`, which receives the exact frozen post-authorization profile, task, assignment, parent identity, and child execution identity before Runtime selects manager versus leaf. +`resolveDeliverable` receives that same context only for backend-derived leaves, allowing each authorized assignment to carry its own completion check while `deliverable` remains the run-wide fallback. +An attached `rootHandle` exposes the live root tree, control signals, cancellation, and raw steering through the root manager's real inbox; delivery returns `false` when that manager has no inbox and fails loudly outside the handle's live binding. The parent reserves that driver's allocation once; the nested manager partitions only that allocation, so nested work is neither double-charged nor allowed to borrow outside its branch. For an external-harness supervisor, the automatic path is local CLI bridge execution: Runtime adds the live coordination MCP under the reserved `agent-runtime-coordination` alias and executes the rest of the profile unchanged through a `bridge` `driverBackend ?? backend`. Before that runtime starts, the spawn journal records a `materialized` receipt containing the authored-profile, effective-profile, and platform-attachment digests. Volatile endpoints, credentials, and process/session bindings are never stored there. Each execution or restart instead appends a digest-only `execution-bound` receipt for that attempt, so the stable profile identity survives restarts while every concrete attempt remains distinguishable. A remote sandbox cannot reach Runtime's loopback coordination server automatically, so it requires an explicit `driveHarness` that supplies its own reachable relay or tunnel. +When recursive external managers need live steering, `resolveDriveHarness(context)` receives each frozen Runtime-owned manager identity and must return a distinct steerable harness session; one shared custom harness remains valid only when it has no live `deliver` inbox. For manager-authored child profiles, the default security policy blocks local MCP processes, every remote MCP host, shell hooks, and ambient connection grants; products opt in by passing an explicit `profileSecurity` policy and remote-host allowlist. The in-process router supervisor is deliberately narrower because it has no harness environment to materialize. @@ -93,7 +97,7 @@ A general "loop" primitive is the single most common modelling error in this rep | I want to… | Use (import) | Do NOT build | |---|---|---| | Run one product chat turn with streamed events, ordered persistence hooks, and stable execution/turn identity | `handleChatTurn(...)` + `deriveExecutionId(...)`: `/durable`; pass the derived id as both `executionId` and `turnId` on initial dispatch | importing the broad package entry from an edge worker, treating `executionId` alone as dispatch idempotency, or rebuilding framing and persistence ordering in the product | -| Run a supervisor toward a goal with default setup | `supervise(profile, task, { budget, backend? })`: `/kernel` | hand-wiring `createSupervisor().run` + `blobs`/`perWorker`/`journal`/`executors`; reaching for lower-level calls before you need a specific counterparty | +| Run a supervisor toward a goal with default setup, trusted per-spawn manager/leaf policy, per-leaf completion checks, or a steerable root | `supervise(profile, task, { budget, backend?, rootHandle?, isDriverProfile?, resolveDeliverable? })`: `/kernel` | hand-wiring `createSupervisor().run` + `blobs`/`perWorker`/`journal`/`executors`; routing product authority through model-authored metadata; reaching for lower-level calls before you need a specific counterparty | | Analyze a settled worker's tool use | `run_analyst` over the worker's persisted `WorkerTraceEvidence`; Runtime reconstructs agent-eval's bounded `TraceAnalysisStore`: `/kernel` + `/mcp` | passing the worker's final prose or result blob to a trace analyst; a worker with no structured tool spans carries an explicit unavailable reason and analysis is refused | | **Supervise agents to solve a graded `AgenticSurface` task** (workers `runAgentic` the surface, settle on its own check, driver self-improves from the failing tests) | `superviseSurface(profile, task, { surface, worker })`: `/kernel` | a worker-seam + a "self-improving supervisor" wrapper around `supervise()`; passing a custom `makeWorkerAgent` that runs `runAgentic` | | Run a profile through a topology shape over the keystone Supervisor, end-to-end | `runPersonified({ persona, shape, task, budget })`: `/kernel` | a hand-rolled `createSupervisor().run` + seam-wiring helper | @@ -144,11 +148,13 @@ A general "loop" primitive is the single most common modelling error in this rep | State any benchmark/A-B claim | `pairedLift(...)` (bench) over `pairedBootstrap`/`heldoutSignificance` (substrate) | your own bootstrap loop/PRNG per gate; a point lift without `low/high/pairs` | | Let an agent **delegate ONE generic INTENT** (no fixed coder/researcher type) and get the result + real spend SYNCHRONOUSLY | the **`delegate` tool**: `createDelegateHandler` via `createMcpServer({ delegateSupervisor })`; mount it over the `agent-runtime mcp` bin with `MCP_ENABLE_DELEGATE=1` (the bin authors a supervisor over a `sandbox` backend): `/mcp` | a hardcoded coder/researcher profile, or task-specific `delegate_code`/`delegate_research` verbs (RETIRED): `delegate` is the ONE delegation path and the only one with a cost channel | | Run a coding task INSIDE the agent's OWN sandbox session (a sibling box, fresh branch, validated patch) | `detachedSessionDelegate({ sandboxClient \| executor, workerProfile? })`: `/mcp` (pass the worker `AgentProfile`; omit for a minimal model-only default) | a hardcoded coder profile baked into the delegate; `delegate()` (that spawns workers in a *chosen* backend, not the agent's own session) | -| Have a **supervisor spawn + live-drive workers in a backend you choose** and observe or steer them while the coordinator is alive | `supervise(profile, task, { backend, runDir, runId })` for the complete path, or the lower-level **coordination MCP** via `createCoordinationTools` / `serveCoordinationMcp` over a live `Scope`; same-host restart restores committed work, exact identities, spend, waits, and the original deadline while preserving prior coordination evidence | `detachedSessionDelegate`, which is own-sandbox-session only and one-shot; same-host restart does not reattach work that was active when its process died, and cross-box durable coordination still requires a transport binding | +| Have a **supervisor spawn + live-drive workers in a backend you choose** and observe or steer them while the coordinator is alive | `supervise(profile, task, { backend, runDir, runId, rootHandle })` for the complete path, or the lower-level **coordination MCP** via `createCoordinationTools` / `serveCoordinationMcp` over a live `Scope`; same-host restart restores committed work, exact identities, spend, waits, and the original deadline while preserving prior coordination evidence | `detachedSessionDelegate`, which is own-sandbox-session only and one-shot; same-host restart does not reattach work that was active when its process died, and cross-box durable coordination still requires a transport binding | +| Cold-read one complete persisted recursive supervision forest | `loadSpawnForest(journal, runId)`: `/kernel`; follows only each spawn's Runtime-attested `ownedTreeRoot` and returns every nested tree, flattened node/event with its tree root, in-doubt node, and missing driver subtree; legacy records without that field remain leaves | reconstructing Runtime's private nested-tree keys, treating the open runtime string `driver` as ownership, or reading only the root tree and presenting it as the whole run | | Stand up a vertical agent in the eval loop | `defineAgent(manifest)` + `createSurfaceImprovementProposer`: `/agent` | a per-vertical manifest parser, surface-validator, or bespoke findings-to-patch mapper | | Observe + deliver Intelligence on a live agent (send RunRecords + receive certified profile/diffs) | `withIntelligence(agent, { project, target })`: `/intelligence` (proposals surfaced, never auto-applied; `effort: 'off'` proves inference-only billing) | a custom trace-wrapper, a second receive path, or hand-rolled effort/tier config | | Turn trace evidence into one measured, review-only agent proposal | `proposeAgentImprovement({ analysis, profile, improvement, buildExperiment, placeCell })` in `/intelligence` (Runtime seals the optimizer ancestry) | manually joining analysis, optimizer ancestry, exact candidate execution, uncertainty, and candidate identity | | Freeze a measured profile/diff plus a content-addressed code surface into one executable candidate | `buildAgentCandidateBundle(...)`, then `verifyAgentCandidateBundle(...)` at execution: `/candidate-execution` | a product callback that converts profile fields, reproduces Git diff flags, hashes bytes, or assembles `AgentCandidateBundle` by hand | +| Convert or validate the exact profile carried by a generic candidate | `freezeGenericAgentCandidateProfile`, `parseExactCandidateProfile`, and `omitUndefinedObjectFields`: root or `/candidate-execution` | copying Runtime's exact-profile conversion logic into a product composer | | Record approve/reject/change-request feedback against one exact proposal | `reviewAgentImprovementProposal(proposal, review)`: `/intelligence` | a mutable status row that is not bound to candidate bytes | | Run and grade the exact signed baseline-versus-candidate matrix before review | `runAgentCandidateExperiment({ experiment, placeCell })` in `/intelligence`; use `createProtectedExactProcessCandidateExperimentExecutor(...)` for any exact-process provider with protected model grants and pass the remaining product ports as `hostPorts` | product-local pairing, retry, isolation, receipt, and comparison code | | Authorize and execute writes only for the exact measured and approved candidate | `createAgentImprovementActivation(...)`, then `executeAgentImprovementActivation(...)` with one idempotent transition in `/intelligence`; opaque profile changes use `prepareAgentImprovementProfileActivation({ stateDigest, resolveState? })`, target one complete profile identity, and restore only from product-retained state by exact digest; use `createKnowledgeImprovementActivationExecutor(...)` from `/knowledge` for one local KB | a mutable approval flag, a second per-surface approval path, a best-effort profile restore, or a write that does not persist an exact result | diff --git a/package.json b/package.json index 9dd097bf..0b48421d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.109.2", + "version": "0.110.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -162,8 +162,8 @@ "license": "MIT", "packageManager": "pnpm@11.17.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.135.3 <0.136.0", - "@tangle-network/agent-interface": ">=0.36.0 <0.37.0", + "@tangle-network/agent-eval": ">=0.135.4 <0.136.0", + "@tangle-network/agent-interface": ">=0.37.0 <0.38.0", "@tangle-network/sandbox": ">=0.15.0 <0.16.0", "playwright": "^1.40.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index afa2f84d..3e97b99f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,17 +10,17 @@ catalogs: specifier: 0.18.5 version: 0.18.5 '@tangle-network/agent-eval': - specifier: 0.135.3 - version: 0.135.3 + specifier: 0.135.4 + version: 0.135.4 '@tangle-network/agent-interface': - specifier: 0.36.0 - version: 0.36.0 + specifier: 0.37.0 + version: 0.37.0 '@tangle-network/agent-knowledge': - specifier: 6.1.11 - version: 6.1.11 + specifier: 7.0.0 + version: 7.0.0 '@tangle-network/agent-profile-materialize': - specifier: 0.9.2 - version: 0.9.2 + specifier: 0.9.3 + version: 0.9.3 '@tangle-network/sandbox': specifier: 0.15.2 version: 0.15.2 @@ -46,10 +46,10 @@ importers: dependencies: '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 6.1.11 + version: 7.0.0 '@tangle-network/agent-profile-materialize': specifier: 'catalog:' - version: 0.9.2(@tangle-network/agent-interface@0.36.0) + version: 0.9.3(@tangle-network/agent-interface@0.37.0) tar-stream: specifier: 3.2.0 version: 3.2.0 @@ -62,10 +62,10 @@ importers: version: 2.5.5 '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.135.3 + version: 0.135.4 '@tangle-network/agent-interface': specifier: 'catalog:' - version: 0.36.0 + version: 0.37.0 '@tangle-network/sandbox': specifier: 'catalog:' version: 0.15.2(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) @@ -113,13 +113,13 @@ importers: dependencies: '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.135.3 + version: 0.135.4 '@tangle-network/agent-interface': specifier: 'catalog:' - version: 0.36.0 + version: 0.37.0 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 6.1.11 + version: 7.0.0 '@tangle-network/agent-runtime': specifier: workspace:* version: link:.. @@ -1111,28 +1111,34 @@ packages: '@tangle-network/agent-core@0.4.25': resolution: {integrity: sha512-3hfIs64b/1/C1ezYhSPu6NC449lO4XAq79uaol0z0eLXcXDM+2znXThTeMTEZUPJ4iAsR514xcHL2OrP3dsE3g==} + '@tangle-network/agent-core@0.4.26': + resolution: {integrity: sha512-Rf4qWZ5qYQP86z71CvniKL16wsxhkzcCNIrVEvOQn6hyEShUMuR+ZhTl61yTPGT0S//VowN9of4e9Uyv/CKItg==} + '@tangle-network/agent-eval@0.135.2': resolution: {integrity: sha512-qgthVcu8gQy57SgKFnx+Oug0t3ooPoqsDEMteNdye+EbkcvDH7k42JJrR4AYDRW0mbfNsA3WdzhlaGAK+Dd+Ew==} engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-eval@0.135.3': - resolution: {integrity: sha512-KVTXinEzGQJ1fQ0cUm6bKEnnD8gGijb6HiyYG28/8vvndvq5cJsBCuPmcCw2wfvEqIUOlS/3ABDOQbxGIiGJRw==} + '@tangle-network/agent-eval@0.135.4': + resolution: {integrity: sha512-v193l98fHp4uuyENATAzsNBnm4ETb+YEIFBrim4NNxRxsRxJfiPB9ny/sbuOY8odfUgCB9xRVdiUrAGu4zxO8A==} engines: {node: '>=20'} hasBin: true '@tangle-network/agent-interface@0.36.0': resolution: {integrity: sha512-l48SiAY5Atx6ZTL86/XP2wSUPfJJ1sga82/F6V0KqwVkE5MHWiFql6wsLoqk0lc7s4EJe6+cKky+2zzxFuxDpg==} - '@tangle-network/agent-knowledge@6.1.11': - resolution: {integrity: sha512-2p5TLMreGGEsYp+V2LWVm1YjTfLlHHaq5cT1qtLmN4Ao7Ku5pnCdptq6sZwf8kbgHUAwPWFuomWw7iXTzKMeiw==} + '@tangle-network/agent-interface@0.37.0': + resolution: {integrity: sha512-jndAeQ5j/Vrb4hBNG/iT4B4DCWJ0sPXw0kuHsBOYe0a8ypYtGlOozGLAtR1s1Dw49YHWxcJQwgsarvwXdSt93w==} + + '@tangle-network/agent-knowledge@7.0.0': + resolution: {integrity: sha512-RNsAIN0itfO6Cci6EqDQ/84DX/nbSdF2jP2WFjR4jLU/xylHaEeaBaSDHGlhJ9EYbracUxMWxzitjki1DnBNdg==} engines: {node: '>=20.19.0'} hasBin: true - '@tangle-network/agent-profile-materialize@0.9.2': - resolution: {integrity: sha512-xHsuJOSwN7FtdHn8liNeExWe45Oj18B5y9Yu+BibDFHUa3hZ8B2lX6hW4mVyHIFasoePn1zGqsGKQ8FzoobX+w==} + '@tangle-network/agent-profile-materialize@0.9.3': + resolution: {integrity: sha512-/WuiAOeP86HpnCXzL7E3xX4hWpuw5dK7MlZyvKLRf7xCGp3w7iTrUXH5mxcaYWpxGHbSC3V8tNpAUGGsMgGSEg==} peerDependencies: - '@tangle-network/agent-interface': '>=0.36.0 <0.37.0' + '@tangle-network/agent-interface': '>=0.36.0 <0.38.0' '@tangle-network/sandbox@0.15.2': resolution: {integrity: sha512-haEdX9shY5jXQABwfipxmkrnz0Po42Dvtk41VTNCh88uw4iXcSgJdGBTBLYoriHZZZfJEenQPjlPjpgj8sm3NQ==} @@ -1816,6 +1822,15 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + spdx-exceptions@2.5.0: + resolution: {integrity: sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==} + + spdx-expression-parse@5.0.0: + resolution: {integrity: sha512-vngmw3Rgn+o2arXNbnZaj5UtOEBuWBfvaI+Wc8GFfykIhA5/vdK9/Sp/XkLv63dykz2rxKDvKEHupF5P0FORcQ==} + + spdx-license-ids@3.0.23: + resolution: {integrity: sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -2762,6 +2777,11 @@ snapshots: '@tangle-network/agent-interface': 0.36.0 zod: 4.4.3 + '@tangle-network/agent-core@0.4.26': + dependencies: + '@tangle-network/agent-interface': 0.37.0 + zod: 4.4.3 + '@tangle-network/agent-eval@0.135.2': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) @@ -2772,13 +2792,13 @@ snapshots: hono: 4.12.32 zod: 4.4.3 - '@tangle-network/agent-eval@0.135.3': + '@tangle-network/agent-eval@0.135.4': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) '@ax-llm/ax': 23.0.5(zod@4.4.3) '@hono/node-server': 2.0.12(hono@4.12.32) - '@tangle-network/agent-core': 0.4.25 - '@tangle-network/agent-interface': 0.36.0 + '@tangle-network/agent-core': 0.4.26 + '@tangle-network/agent-interface': 0.37.0 hono: 4.12.32 zod: 4.4.3 @@ -2787,16 +2807,22 @@ snapshots: '@noble/hashes': 1.8.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@6.1.11': + '@tangle-network/agent-interface@0.37.0': + dependencies: + '@noble/hashes': 1.8.0 + spdx-expression-parse: 5.0.0 + zod: 4.4.3 + + '@tangle-network/agent-knowledge@7.0.0': dependencies: '@tangle-network/agent-eval': 0.135.2 '@tangle-network/agent-interface': 0.36.0 proper-lockfile: 4.1.2 zod: 4.4.3 - '@tangle-network/agent-profile-materialize@0.9.2(@tangle-network/agent-interface@0.36.0)': + '@tangle-network/agent-profile-materialize@0.9.3(@tangle-network/agent-interface@0.37.0)': dependencies: - '@tangle-network/agent-interface': 0.36.0 + '@tangle-network/agent-interface': 0.37.0 '@tangle-network/sandbox@0.15.2(viem@2.54.6(typescript@6.0.3)(zod@4.4.3))': dependencies: @@ -3460,6 +3486,15 @@ snapshots: source-map-js@1.2.1: {} + spdx-exceptions@2.5.0: {} + + spdx-expression-parse@5.0.0: + dependencies: + spdx-exceptions: 2.5.0 + spdx-license-ids: 3.0.23 + + spdx-license-ids@3.0.23: {} + stackback@0.0.2: {} std-env@4.2.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 370a926a..94c54844 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -18,10 +18,10 @@ allowBuilds: catalog: '@arethetypeswrong/cli': 0.18.5 '@types/node': 26.1.1 - '@tangle-network/agent-eval': 0.135.3 - '@tangle-network/agent-interface': 0.36.0 - '@tangle-network/agent-knowledge': 6.1.11 - '@tangle-network/agent-profile-materialize': 0.9.2 + '@tangle-network/agent-eval': 0.135.4 + '@tangle-network/agent-interface': 0.37.0 + '@tangle-network/agent-knowledge': 7.0.0 + '@tangle-network/agent-profile-materialize': 0.9.3 '@tangle-network/sandbox': 0.15.2 publint: 0.3.22 tsdown: 0.22.14 diff --git a/src/candidate-execution/index.ts b/src/candidate-execution/index.ts index 794b4bd7..3a5461eb 100644 --- a/src/candidate-execution/index.ts +++ b/src/candidate-execution/index.ts @@ -61,8 +61,11 @@ export { agentCandidateProfileAsAgentProfile, applyExactAgentProfileDiff, assertCandidateProfileBinding, + freezeGenericAgentCandidateProfile, + omitUndefinedObjectFields, parseExactAgentProfile, parseExactAgentProfileDiff, + parseExactCandidateProfile, } from './profile' export { type AgentCandidateModelGrantActivateInput, diff --git a/src/candidate-execution/profile.ts b/src/candidate-execution/profile.ts index 93a94780..e9f34275 100644 --- a/src/candidate-execution/profile.ts +++ b/src/candidate-execution/profile.ts @@ -213,6 +213,7 @@ export function applyExactAgentProfileDiff( return parseExactAgentProfile(applied, `${label} result`) } +/** Parse a candidate profile without silently discarding unsupported or non-canonical fields. */ export function parseExactCandidateProfile(input: unknown): AgentCandidateProfile { const parsed = agentCandidateProfileSchema.parse(input) assertCanonicalParse(input, parsed, 'candidate profile') @@ -302,6 +303,7 @@ export function agentCandidateProfileAsAgentProfile( return output as AgentProfile } +/** Recursively remove undefined object fields while refusing undefined array entries. */ export function omitUndefinedObjectFields(value: unknown, path: string): unknown { if (Array.isArray(value)) { return value.map((entry, index) => { diff --git a/src/durable/spawn-journal.ts b/src/durable/spawn-journal.ts index 160510bf..1f6aefcf 100644 --- a/src/durable/spawn-journal.ts +++ b/src/durable/spawn-journal.ts @@ -20,7 +20,9 @@ * @experimental */ +import { detachedSnapshot } from '../runtime/supervise/snapshot' import { workerTraceAnalysisStore } from '../runtime/supervise/trace-evidence' +import { nestedDriverTreeRoot } from '../runtime/supervise/tree-key' import type { NodeExecutionIdentity, NodeId, @@ -41,6 +43,54 @@ import { parseCommittedJsonLines, prepareJsonlAppend, writeAllBytes } from './js export { contentAddress } from './content-address' +/** One journal tree in a recursively loaded supervision forest. */ +export interface SpawnForestTree { + readonly root: NodeId + /** Driver node that owns this tree; absent for the requested root tree. */ + readonly ownerNodeId?: NodeId + /** Journal tree containing `ownerNodeId`; absent for the requested root tree. */ + readonly parentTreeRoot?: NodeId + readonly events: ReadonlyArray + readonly view: TreeView +} + +/** One event with the journal tree that establishes its cursor namespace. */ +export interface SpawnForestEvent { + readonly treeRoot: NodeId + readonly event: SpawnEvent +} + +/** One flattened node with the journal tree that owns its records. */ +export interface SpawnForestNode extends NodeSnapshot { + readonly treeRoot: NodeId +} + +/** A spawned worker with no terminal record in a cold snapshot. Resume treats the same state as + * in-doubt and conservatively retains its reservation. Root nodes and armed waits are excluded. */ +export interface SpawnForestInDoubtNode { + readonly treeRoot: NodeId + readonly nodeId: NodeId + readonly label: string + readonly runtime: Runtime +} + +/** A driver spawn whose owned journal tree was never begun before the process stopped. */ +export interface SpawnForestMissingTree { + readonly parentTreeRoot: NodeId + readonly ownerNodeId: NodeId + readonly root: NodeId +} + +/** Complete cold-readable view of one recursive supervision run. */ +export interface SpawnForest { + readonly root: NodeId + readonly trees: ReadonlyArray + readonly nodes: ReadonlyArray + readonly events: ReadonlyArray + readonly inDoubt: ReadonlyArray + readonly missingTrees: ReadonlyArray +} + // ── Content addressing ────────────────────────────────────────────────────── /** @@ -259,6 +309,116 @@ export class FileSpawnJournal implements SpawnJournal { } } +/** + * Load every journal tree owned by one recursive supervision run and flatten its nodes/events. + * + * Nested driver tree keys are a Runtime implementation detail; callers should use this reader + * instead of deriving or scanning keys themselves. The reader follows only the explicit + * `ownedTreeRoot` written after Runtime privately attested a recursive executor; the open runtime + * string `driver` is never treated as ownership. Legacy records without `ownedTreeRoot` are + * intentionally treated as leaves rather than guessing or scanning convention-derived keys. + * This preserves each tree's independent cursor namespace on flattened events. + * A driver whose subtree was never begun is reported in `missingTrees`; any spawned non-root node + * without a terminal record is reported in `inDoubt`, matching resume's conservative lost-work + * interpretation. + * + * This is a cold/quiescent reader, not a transaction across an actively mutating file. Every value + * returned is a detached immutable snapshot, so later journal writes or caller mutation cannot + * change the result already observed. + */ +export async function loadSpawnForest(journal: SpawnJournal, root: NodeId): Promise { + const rootEvents = await journal.loadTree(root) + if (rootEvents === undefined) { + throw new Error(`loadSpawnForest: no journaled tree for root '${root}'`) + } + + const trees: SpawnForestTree[] = [] + const nodes: SpawnForestNode[] = [] + const events: SpawnForestEvent[] = [] + const inDoubt: SpawnForestInDoubtNode[] = [] + const missingTrees: SpawnForestMissingTree[] = [] + const visited = new Set() + const queue: Array<{ + readonly root: NodeId + readonly events: SpawnEvent[] + readonly ownerNodeId?: NodeId + readonly parentTreeRoot?: NodeId + }> = [{ root, events: rootEvents }] + + for (let cursor = 0; cursor < queue.length; cursor += 1) { + const current = queue[cursor]! + if (visited.has(current.root)) { + throw new Error(`loadSpawnForest: recursive journal tree '${current.root}' was reached twice`) + } + visited.add(current.root) + const stableEvents = detachedSnapshot( + current.events, + `loadSpawnForest tree ${JSON.stringify(current.root)}`, + ) + const view = detachedSnapshot( + materializeTreeView([...stableEvents]), + `loadSpawnForest view ${JSON.stringify(current.root)}`, + ) + trees.push({ + root: current.root, + ...(current.ownerNodeId === undefined ? {} : { ownerNodeId: current.ownerNodeId }), + ...(current.parentTreeRoot === undefined ? {} : { parentTreeRoot: current.parentTreeRoot }), + events: stableEvents, + view, + }) + nodes.push(...view.nodes.map((node) => ({ treeRoot: current.root, ...node }))) + events.push(...stableEvents.map((event) => ({ treeRoot: current.root, event }))) + + const terminal = new Set() + for (const event of stableEvents) { + if (event.kind === 'settled' || event.kind === 'cancelled') terminal.add(event.id) + } + const spawns = stableEvents + .filter( + (event): event is Extract => event.kind === 'spawned', + ) + .sort((a, b) => a.seq - b.seq || a.id.localeCompare(b.id)) + for (const spawn of spawns) { + if (spawn.parent !== undefined && !terminal.has(spawn.id)) { + inDoubt.push({ + treeRoot: current.root, + nodeId: spawn.id, + label: spawn.label, + runtime: spawn.runtime, + }) + } + if (spawn.ownedTreeRoot === undefined) continue + const expectedRoot = nestedDriverTreeRoot(current.root, spawn.id) + if (spawn.ownedTreeRoot !== expectedRoot) { + throw new Error( + `loadSpawnForest: node '${spawn.id}' owns non-canonical tree '${spawn.ownedTreeRoot}'; expected '${expectedRoot}'`, + ) + } + const nestedRoot = spawn.ownedTreeRoot + const nestedEvents = await journal.loadTree(nestedRoot) + if (nestedEvents === undefined) { + missingTrees.push({ + parentTreeRoot: current.root, + ownerNodeId: spawn.id, + root: nestedRoot, + }) + continue + } + queue.push({ + root: nestedRoot, + events: nestedEvents, + ownerNodeId: spawn.id, + parentTreeRoot: current.root, + }) + } + } + + return detachedSnapshot( + { root, trees, nodes, events, inDoubt, missingTrees }, + 'loadSpawnForest result', + ) +} + type SpawnJournalRecord = | { kind: 'begin'; root: NodeId; at: string } | { kind: 'event'; root: NodeId; event: SpawnEvent } @@ -582,6 +742,7 @@ export function materializeTreeView(events: SpawnEvent[]): TreeView { status: 'pending', runtime: ev.runtime, budget: ev.budget, + ...(ev.ownedTreeRoot === undefined ? {} : { ownedTreeRoot: ev.ownedTreeRoot }), ...(ev.assignmentId === undefined ? {} : { assignmentId: ev.assignmentId }), ...(ev.identity ? { identity: ev.identity } : {}), spent: zeroSpend(), @@ -670,6 +831,7 @@ interface MutableSnapshot { status: NodeStatus runtime: Runtime budget: NodeSnapshot['budget'] + ownedTreeRoot?: NodeSnapshot['ownedTreeRoot'] assignmentId?: string identity?: NodeSnapshot['identity'] materialization?: NodeSnapshot['materialization'] @@ -712,6 +874,7 @@ function freezeSnapshot(node: MutableSnapshot): NodeSnapshot { status: node.status, runtime: node.runtime, budget: node.budget, + ownedTreeRoot: node.ownedTreeRoot, assignmentId: node.assignmentId, identity: node.identity, materialization: node.materialization, diff --git a/src/runtime/index.ts b/src/runtime/index.ts index e74ee3d8..c9cecd0d 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -39,12 +39,19 @@ export { FileSpawnJournal, InMemoryResultBlobStore, InMemorySpawnJournal, + loadSpawnForest, materializeTreeView, // The waits a journaled tree shows as armed but never woken — what a resumed run re-arms with // the ORIGINAL deadline. Exported for the same reason the replay readers are: a durable wait a // consumer cannot read back is only a log line. pendingWaits, replaySpawnTree, + type SpawnForest, + type SpawnForestEvent, + type SpawnForestInDoubtNode, + type SpawnForestMissingTree, + type SpawnForestNode, + type SpawnForestTree, } from '../durable/spawn-journal' // The typed coordination-bus event (up: settled/question/finding; authorized instruction receipt; // down: steer/answer delivery outcome) — surfaced here so a host folding the bus onto its own timeline can @@ -695,7 +702,9 @@ export { // from a backend config + an optional completion oracle (settled⟺delivered). export { type AuthorizedSpawn, + type AuthorizedSpawnContext, DEFAULT_AUTHORED_PROFILE_SECURITY_POLICY, + type DeliverableResolutionInput, type SuperviseOptions, supervise, workerFromBackend, @@ -706,7 +715,9 @@ export { createSupervisor } from './supervise/supervisor' // a coding-CLI harness → a sandboxed harness driving the coordination verbs. No hand-built brain. export { type DriveHarness, + type DriveHarnessOwnerContext, type ObserveSupervisorNodeEvent, + type ResolveDriveHarness, type ResolveSupervisorTools, type SupervisorAgentDeps, type SupervisorNodeContext, @@ -775,6 +786,7 @@ export type { SpawnPrior, SpawnRejection, Spend, + SteerableRootHandle, SupervisedResult, Supervisor, SupervisorOpts, diff --git a/src/runtime/supervise/coordination-driver.ts b/src/runtime/supervise/coordination-driver.ts index e002026d..bedd8de2 100644 --- a/src/runtime/supervise/coordination-driver.ts +++ b/src/runtime/supervise/coordination-driver.ts @@ -54,6 +54,7 @@ import { runTree, type SupervisorFinalizer, } from './finalizer' +import { createInbox, type Inbox } from './inbox' import { createProgressTracker, type ProgressTracker, @@ -175,6 +176,9 @@ export interface DriverAgentOptions { * the delivered-only invariant (`runFinalizer`): whatever the finalizer, an undelivered or * invalid child's output stays unreachable. */ readonly finalizer?: SupervisorFinalizer + /** Optional shared manager inbox used by a wrapper that must accept messages before async node + * setup finishes. Ordinary callers omit it and the driver owns a fresh inbox. */ + readonly inbox?: Inbox } /** The default chapter-close prompt: the brain summarizes its OWN progress for its future self before @@ -322,9 +326,13 @@ export function driverAgent(opts: DriverAgentOptions): Agent { // pure anti-runaway guard, NOT the intended limit. const maxTurns = opts.maxTurns === 0 ? runawayTripwireTurns : (opts.maxTurns ?? 16) const now = opts.now ?? Date.now + const inbox = opts.inbox ?? createInbox() return { name: opts.name, + deliver(message): boolean { + return inbox.deliver(message) + }, async act(task, scope: Scope): Promise { const coord = createCoordinationTools({ scope, @@ -489,6 +497,12 @@ export function driverAgent(opts: DriverAgentOptions): Agent { // that can no longer spawn (pool starved) or has run past the deadline stops here instead of // burning turns. Checked before each inference turn. hooks: { + beforeTurn: (_turn, messages) => { + const pending = inbox.drain() + if (pending.length > 0) { + messages.push({ role: 'user', content: inbox.fold(pending) }) + } + }, stopBefore: () => { // HARD CEILINGS FIRST, and independently — a progress rule may never keep a run alive // past one, so they are not folded into the same expression. diff --git a/src/runtime/supervise/driver-executor.ts b/src/runtime/supervise/driver-executor.ts index eadf5846..7fa9c409 100644 --- a/src/runtime/supervise/driver-executor.ts +++ b/src/runtime/supervise/driver-executor.ts @@ -47,6 +47,7 @@ import { type NestedScopeSeam, nestedScopeSeamKey, } from './scope' +import { attestNestedDriverTreeOwner, driverRuntime, nestedDriverTreeRoot } from './tree-key' import type { Agent, AgentExecutionRef, @@ -65,7 +66,7 @@ import type { } from './types' /** The runtime tag the registry maps a driver child to. */ -export const driverRuntime = 'driver' as const +export { driverRuntime } from './tree-key' /** The metadata marker on a driver child's spec the recursive registry routes on. */ const driverRole = 'driver' @@ -105,9 +106,17 @@ export function driverChild( driver: driver as Agent, journal, } + const deliver = driver.deliver?.bind(driver) return { name, executorSpec: spec, + ...(deliver + ? { + deliver(message: unknown): boolean { + return deliver(message) !== false + }, + } + : {}), act(): Promise { throw new ValidationError( `driverChild: "${name}" was run directly; a driver child runs through its nested-scope executor`, @@ -148,6 +157,7 @@ export const driverExecutorFactory: ExecutorFactory = (spec, ctx) => { ) } const driver = spec.driver + const deliver = driver.deliver?.bind(driver) const journal = spec.journal const seam = readNestedScopeSeam(ctx) @@ -173,10 +183,17 @@ export const driverExecutorFactory: ExecutorFactory = (spec, ctx) => { const executor: Executor = { runtime: driverRuntime, + ...(deliver + ? { + deliver(message: unknown): boolean { + return deliver(message) !== false + }, + } + : {}), async execute(task, signal): Promise> { // The nested tree key namespaces this driver's children inside the ONE shared // journal, so its cursor seqs never collide with the parent's per-tree guard. - const nestedRoot = nestedTreeKey(seam) + const nestedRoot = nestedDriverTreeRoot(seam.journalRoot, seam.nodeId) await journal.beginTree(nestedRoot, new Date(0).toISOString()) const controller = new AbortController() @@ -208,12 +225,11 @@ export const driverExecutorFactory: ExecutorFactory = (spec, ctx) => { reported: childWork, reservation: addSpend(childWork, meteredSpend ?? zeroSpend()), } - // Completion-oracle propagation: a driver "delivered" iff at least one of its DIRECT - // children settled `valid` (the child its keep-best finalize returns). Deriving the - // driver child's verdict this way composes delivery UP the recursion — a sub-driver is - // `valid` only when it itself selected a delivered child — so a node never settles - // "done = delivered" on a sub-tree that delivered nothing (Foreman's 0/18 lesson). - const verdict = deriveDeliveryVerdict(settled) + // Completion propagation follows both facts: the subtree delivered at least one valid + // child, and this manager's finalizer actually accepted an output. A custom finalizer may + // refuse an otherwise valid child because product state is incomplete; that refusal must + // remain invalid when the nested manager settles into its parent. + const verdict = deriveDeliveryVerdict(settled, out) artifact = { outRef: `${driverRuntime}:${nestedRoot}`, out, @@ -254,10 +270,11 @@ export const driverExecutorFactory: ExecutorFactory = (spec, ctx) => { return artifact }, } + const treeOwner = attestNestedDriverTreeOwner(executor) const ownerRuntime = runtimeOwnedScopeOwnerRuntime(driver) return ownerRuntime === undefined - ? executor - : attestRuntimeOwnedDeferredExecutor(executor, ownerRuntime) + ? treeOwner + : attestRuntimeOwnedDeferredExecutor(treeOwner, ownerRuntime) } /** @@ -281,12 +298,6 @@ export function withDriverExecutor(base: ExecutorRegistry): ExecutorRegistry { // ── Helpers ────────────────────────────────────────────────────────────────────── -/** Derive the nested-tree key from the parent's durable journal root and this driver child's stable - * node id. Sibling node ids are unique within the parent tree, and replay derives the same key. */ -function nestedTreeKey(seam: NestedScopeSeam): string { - return `${seam.journalRoot}/${seam.nodeId}` -} - async function closeNestedScope( scope: Scope, controller: AbortController, @@ -463,13 +474,15 @@ async function safeRollup( } } -/** Derive the driver child's delivery verdict from its DIRECT children's settlements: - * `valid` iff any direct child settled `done` AND `valid` (the keep-best finalize's pick); +/** Derive the driver child's delivery verdict from its DIRECT children's settlements and the + * manager's actual finalized output: `valid` iff a direct child delivered AND the finalizer + * returned a defined output; * `score` = the best delivered score. Returns `undefined` when no child settled at all (the * driver itself produced nothing to bubble a verdict from). Fail-closed: a child whose verdict * carried no `valid` counts as not-delivered. */ function deriveDeliveryVerdict( settled: ReadonlyArray<{ status: 'done' | 'down'; verdict?: DefaultVerdict }>, + finalizedOutput: unknown, ): DefaultVerdict | undefined { let sawChild = false let anyValid = false @@ -490,9 +503,10 @@ function deriveDeliveryVerdict( } } if (!sawChild) return undefined + const accepted = anyValid && finalizedOutput !== undefined return { - valid: anyValid, - score: anyValid ? (bestValidScore ?? 1) : (bestDoneScore ?? 0), + valid: accepted, + score: accepted ? (bestValidScore ?? 1) : (bestDoneScore ?? 0), } } diff --git a/src/runtime/supervise/inbox.ts b/src/runtime/supervise/inbox.ts index 0de13105..adf161a6 100644 --- a/src/runtime/supervise/inbox.ts +++ b/src/runtime/supervise/inbox.ts @@ -11,7 +11,8 @@ * in-flight turn immediately, then re-plan with the message folded in — breaking the worker out * of a wrong path mid-task instead of waiting for it to finish the step. * - * `deliver` never throws — a malformed message is ignored, per the `Executor.deliver` contract. + * `deliver` never throws — a malformed message is ignored and returns `false`, so no caller can + * report delivery for bytes this inbox discarded. * * @experimental */ @@ -26,8 +27,9 @@ export interface InboxMessage { } export interface Inbox { - /** The `Executor.deliver` implementation — accept a raw down-message from `Scope.send`. */ - deliver(msg: unknown): void + /** The `Executor.deliver` implementation. Returns false when the raw message is malformed and + * therefore was not queued; callers must not acknowledge a message this inbox discarded. */ + deliver(msg: unknown): boolean /** Remove and return all pending messages (the flush). */ drain(): InboxMessage[] pending(): number @@ -60,10 +62,11 @@ export function createInbox(): Inbox { return { deliver(msg) { const m = parseDown(msg) - if (!m) return + if (!m) return false pending.push(m) // A forceful message aborts the turn currently in flight (if any). if (m.interrupt && live && !live.signal.aborted) live.abort() + return true }, drain() { return pending.splice(0, pending.length) diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index dbe400b1..e98f9cfa 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -62,6 +62,7 @@ import { import { detachedSnapshot } from './snapshot' import { captureWorkerTraceEvidence } from './trace-evidence' import type { TraceSource } from './trace-source' +import { runtimeOwnedNestedDriverTreeRoot } from './tree-key' import type { Agent, AgentSpec, @@ -202,6 +203,7 @@ interface LiveChild { readonly id: NodeId status: NodeStatus runtime: NodeSnapshot['runtime'] + readonly ownedTreeRoot?: NodeId readonly budget: Budget readonly label: string readonly assignmentId?: string @@ -229,7 +231,7 @@ interface LiveChild { /** True once `next()` has yielded this child's settlement. */ delivered: boolean /** The executor's out-of-band inbox, captured at spawn — backs `scope.send`. */ - readonly deliver?: (msg: unknown) => void + readonly deliver?: (msg: unknown) => boolean /** The executor's optional live progress read, captured at spawn — backs `scope.progress`. */ readonly readProgress?: () => ExecutorProgress | undefined /** The executor's optional live tool trace, captured at spawn — backs `scope.traceSource`. */ @@ -612,6 +614,7 @@ export function createScope(args: ScopeArgs): Scope { }, } const executor = resolved.value(spec, ctx) as Executor + const ownedTreeRoot = runtimeOwnedNestedDriverTreeRoot(executor, args.root, id) const handle: Handle = { id, @@ -637,6 +640,7 @@ export function createScope(args: ScopeArgs): Scope { id, status: 'acquiring', runtime: executor.runtime, + ...(ownedTreeRoot === undefined ? {} : { ownedTreeRoot }), ...(identity ? { identity } : {}), budget: opts.budget, label: opts.label, @@ -650,7 +654,9 @@ export function createScope(args: ScopeArgs): Scope { executionBindings: [], startedAt, lastActivityAt: startedAt, - ...(executor.deliver ? { deliver: executor.deliver.bind(executor) } : {}), + ...(executor.deliver + ? { deliver: (message: unknown): boolean => executor.deliver?.(message) !== false } + : {}), ...(executor.progress ? { readProgress: executor.progress.bind(executor) } : {}), ...(executor.traceSource ? { readTraceSource: executor.traceSource.bind(executor) } : {}), } @@ -672,6 +678,7 @@ export function createScope(args: ScopeArgs): Scope { ...(opts.assignmentId === undefined ? {} : { assignmentId: opts.assignmentId }), budget: opts.budget, runtime: executor.runtime, + ...(ownedTreeRoot === undefined ? {} : { ownedTreeRoot }), ...(identity ? { identity } : {}), seq: ordinal, at: new Date(now()).toISOString(), @@ -858,7 +865,8 @@ export function createScope(args: ScopeArgs): Scope { // Deliver only to a child that is still LIVE (not yet yielded by the cursor) and whose executor // accepts an inbox. A settled/unknown child, or a leaf with no `deliver`, cannot be steered. if (!child || child.delivered || !child.deliver) return false - child.deliver(msg) + const accepted = child.deliver(msg) !== false + if (!accepted) return false // A delivered steer IS activity: it resets the idle clock so a worker that was about to read // as stalled is not immediately re-steered before it can act on the message it just got. child.lastActivityAt = now() @@ -1884,6 +1892,7 @@ function makeTreeView(root: NodeId, children: Map): TreeView status: c.status, runtime: c.runtime, budget: c.budget, + ...(c.ownedTreeRoot === undefined ? {} : { ownedTreeRoot: c.ownedTreeRoot }), ...(c.assignmentId === undefined ? {} : { assignmentId: c.assignmentId }), ...(c.identity ? { identity: c.identity } : {}), ...(c.materialization ? { materialization: c.materialization } : {}), diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index a46bd6e8..66be1827 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -72,6 +72,8 @@ import type { StopRule } from './stop-rules' import { createSupervisor } from './supervisor' import { type DriveHarness, + type DriveHarnessOwnerContext, + type ResolveDriveHarness, type ResolveSupervisorTools, type SupervisorNodeContext, type SupervisorProfile, @@ -82,9 +84,11 @@ import type { AgentExecutionRef, AgentSpec, Budget, + Executor, ExecutorContext, NodeExecutionIdentity, ResultBlobStore, + RootHandle, SpawnJournal, UsageEvent, } from './types' @@ -220,6 +224,7 @@ function driveHarnessFromBackend( const capturedBackend = captureReusableExecutorConfig(backend, 'driveHarnessFromBackend') const boundBackend = bindReusableExecutorExecutionId(capturedBackend, executionId) const baseFactory = createExecutor(boundBackend) + let activeExecutor: Executor | undefined const drive: DriveHarness = async ({ profile, task, @@ -266,6 +271,7 @@ function driveHarnessFromBackend( node: scopeOwnerExecutorNodeContext(scope), seams: {}, }) + activeExecutor = executor let completed = false let started = false let terminalAccountingCaptured = false @@ -425,9 +431,15 @@ function driveHarnessFromBackend( failure = error } } + if (activeExecutor === executor) activeExecutor = undefined } if (failed) throw failure } + drive.deliver = (message): boolean => { + const deliver = activeExecutor?.deliver + if (!deliver) return false + return deliver.call(activeExecutor, message) !== false + } return attestRuntimeOwnedScopeOwner(drive, 'cli') } @@ -443,6 +455,9 @@ function isAsyncIterable(value: unknown): value is AsyncIterable { export interface SuperviseOptions { /** The conserved compute pool for the whole run. */ readonly budget: Budget + /** Caller-created live handle for observing, steering, or cancelling this root manager. Runtime + * attaches it before execution and detaches it after the join barrier. */ + readonly rootHandle?: RootHandle /** Caller-owned cancellation for the complete recursive run. Aborting it cascades through the * root scope and every live child, including acquisition and backend execution. */ readonly signal?: AbortSignal @@ -455,6 +470,12 @@ export interface SuperviseOptions { * without it the supervisor trusts a worker's self-report — exactly the "ran but didn't deliver" * failure mode of a static orchestrator. */ readonly deliverable?: DeliverableSpec + /** Resolve the completion check for one exact authorized backend-derived leaf. The callback runs + * after spawn authorization and driver classification, receives a detached immutable context, + * and may return `undefined` to use the run-wide `deliverable`. Driver profiles never call it. */ + readonly resolveDeliverable?: ( + input: DeliverableResolutionInput, + ) => DeliverableSpec | undefined /** Override the worker seam directly (tests / advanced) instead of deriving it from `backend`. * This is caller-owned execution: profile security, spawn authorization, and recursive-driver * selection below apply only to the backend-derived worker path. `authorizeMessage` still @@ -497,12 +518,10 @@ export interface SuperviseOptions { }, ) => AuthorizedDownMessage /** Decide whether an authorized child becomes another supervisor. By default only - * `metadata.role === 'driver'` does; products may bind this to their own authority record. */ - readonly isDriverProfile?: (input: { - readonly profile: AgentProfile - readonly parent: AgentProfile - readonly depth: number - }) => boolean + * `metadata.role === 'driver'` does. Products receive the same frozen post-authorization + * context as `resolveDeliverable`, so trusted execution/assignment authority can override + * model-authored metadata without a side channel. */ + readonly isDriverProfile?: (input: AuthorizedSpawnContext) => boolean /** The supervisor's router substrate (`profile.harness` omitted or `cli-base`). The profile's * model wins. */ readonly router?: RouterConfig @@ -511,8 +530,12 @@ export interface SuperviseOptions { /** Run an external-harness supervisor explicitly. Required for a remote sandbox; optional as a * caller-owned override for a local bridge. */ readonly driveHarness?: DriveHarness - /** Required with a custom `driveHarness`: declares which complete AgentProfile axes that path - * really applies. Built-in bridge driving supplies its own full-profile contract. */ + /** Resolve one custom external-harness session per trusted manager identity. Use this instead of + * `driveHarness` when recursive managers must be independently steerable. */ + readonly resolveDriveHarness?: ResolveDriveHarness + /** Required with a custom `driveHarness` or `resolveDriveHarness`: declares which complete + * AgentProfile axes that path really applies. Built-in bridge driving supplies its own + * full-profile contract. */ readonly driveHarnessMaterialization?: ProfileMaterializationContract /** Resolve product-owned tools from the exact trusted manager context. The same descriptors and * handlers are bound to router and external-harness managers; resolution happens once per node. */ @@ -641,6 +664,40 @@ export interface AuthorizedSpawn { readonly execution?: AgentExecutionRef } +/** Exact trusted context after a manager-authored spawn has passed product authorization. */ +export interface AuthorizedSpawnContext { + readonly profile: AgentProfile + readonly parent: AgentProfile + readonly parentIdentity: NodeExecutionIdentity + readonly execution: NodeExecutionIdentity + readonly parentNodeId: string + readonly assignmentId: string + readonly task: unknown + readonly budget: Budget + readonly label: string + readonly key?: string + readonly depth: number +} + +/** Exact trusted context for selecting one backend-derived leaf's completion check. */ +export type DeliverableResolutionInput = AuthorizedSpawnContext + +function captureDeliverable( + deliverable: DeliverableSpec, + context: string, +): DeliverableSpec { + if (typeof deliverable !== 'object' || deliverable === null || Array.isArray(deliverable)) { + throw new ValidationError(`${context}: deliverable must be an object`) + } + if (typeof deliverable.check !== 'function') { + throw new ValidationError(`${context}: deliverable.check must be a function`) + } + return Object.freeze({ + ...detachedSnapshot({ describe: deliverable.describe }, `${context} configuration`), + check: deliverable.check, + }) +} + /** Capture the public one-call configuration before any asynchronous work starts. Decision data is * detached and frozen; executable ports are copied as the exact references selected at intake. * Service internals intentionally remain live, while replacing a callback/service on the caller's @@ -650,6 +707,7 @@ function captureSuperviseOptions(opts: SuperviseOptions): SuperviseOptions { backend, driverBackend, deliverable, + resolveDeliverable, router, compaction, watchWorkers, @@ -663,6 +721,7 @@ function captureSuperviseOptions(opts: SuperviseOptions): SuperviseOptions { isDriverProfile, brain, driveHarness, + resolveDriveHarness, resolveSupervisorTools, onCoordinationEvent, executeExtraTool, @@ -671,6 +730,7 @@ function captureSuperviseOptions(opts: SuperviseOptions): SuperviseOptions { finalizer, now, signal, + rootHandle, ...decisionData } = opts const capturedData = detachedSnapshot(decisionData, 'supervise options') @@ -678,15 +738,7 @@ function captureSuperviseOptions(opts: SuperviseOptions): SuperviseOptions { const capturedDriverBackend = driverBackend === undefined ? undefined : snapshotExecutorConfig(driverBackend) const capturedDeliverable = - deliverable === undefined - ? undefined - : Object.freeze({ - ...detachedSnapshot( - { describe: deliverable.describe }, - 'supervise deliverable configuration', - ), - check: deliverable.check, - }) + deliverable === undefined ? undefined : captureDeliverable(deliverable, 'supervise deliverable') const capturedRouter = router === undefined ? undefined @@ -734,6 +786,7 @@ function captureSuperviseOptions(opts: SuperviseOptions): SuperviseOptions { ...(capturedBackend === undefined ? {} : { backend: capturedBackend }), ...(capturedDriverBackend === undefined ? {} : { driverBackend: capturedDriverBackend }), ...(capturedDeliverable === undefined ? {} : { deliverable: capturedDeliverable }), + ...(resolveDeliverable === undefined ? {} : { resolveDeliverable }), ...(capturedRouter === undefined ? {} : { router: capturedRouter }), ...(capturedCompaction === undefined ? {} : { compaction: capturedCompaction }), ...(capturedWatchWorkers === undefined ? {} : { watchWorkers: capturedWatchWorkers }), @@ -747,6 +800,7 @@ function captureSuperviseOptions(opts: SuperviseOptions): SuperviseOptions { ...(isDriverProfile === undefined ? {} : { isDriverProfile }), ...(brain === undefined ? {} : { brain }), ...(driveHarness === undefined ? {} : { driveHarness }), + ...(resolveDriveHarness === undefined ? {} : { resolveDriveHarness }), ...(resolveSupervisorTools === undefined ? {} : { resolveSupervisorTools }), ...(onCoordinationEvent === undefined ? {} : { onCoordinationEvent }), ...(executeExtraTool === undefined ? {} : { executeExtraTool }), @@ -755,6 +809,7 @@ function captureSuperviseOptions(opts: SuperviseOptions): SuperviseOptions { ...(finalizer === undefined ? {} : { finalizer }), ...(now === undefined ? {} : { now }), ...(signal === undefined ? {} : { signal }), + ...(rootHandle === undefined ? {} : { rootHandle }), }) } @@ -891,6 +946,11 @@ export function supervise(profile: SupervisorProfile, task: unknown, opts: Super 'supervise: authorizeSpawn cannot be combined with caller-owned makeWorkerAgent; wrap and authorize the custom factory explicitly or use backend-derived workers', ) } + if (options.makeWorkerAgent && options.resolveDeliverable) { + throw new ValidationError( + 'supervise: resolveDeliverable applies only to backend-derived workers; wrap a caller-owned makeWorkerAgent with its completion checks explicitly', + ) + } const authorizeDownFor = ( parent: AgentProfile, depth: number, @@ -965,12 +1025,16 @@ export function supervise(profile: SupervisorProfile, task: unknown, opts: Super } : undefined const managerBackend = options.driverBackend ?? options.backend - if (options.driveHarness && !options.driveHarnessMaterialization) { + if (options.driveHarness && options.resolveDriveHarness) { + throw new ValidationError('supervise: provide driveHarness or resolveDriveHarness, not both') + } + const hasCustomDriveHarness = Boolean(options.driveHarness || options.resolveDriveHarness) + if (hasCustomDriveHarness && !options.driveHarnessMaterialization) { throw new ValidationError( - 'supervise: custom driveHarness requires driveHarnessMaterialization so profile fields cannot be silently dropped', + 'supervise: custom driveHarness or resolveDriveHarness requires driveHarnessMaterialization so profile fields cannot be silently dropped', ) } - const driverMaterialization = options.driveHarness + const driverMaterialization = hasCustomDriveHarness ? options.driveHarnessMaterialization : managerBackend && automaticDriverBackendSupported(managerBackend) ? backendProfileMaterialization(managerBackend) @@ -978,22 +1042,77 @@ export function supervise(profile: SupervisorProfile, task: unknown, opts: Super if ( isExternalSupervisor(canonicalProfile) && !options.driveHarness && + !options.resolveDriveHarness && (!managerBackend || !automaticDriverBackendSupported(managerBackend)) ) { throw new ValidationError( - `supervise: external supervisor profile.harness=${JSON.stringify(canonicalProfile.harness)} requires a local bridge driverBackend or an explicit driveHarness with reachable coordination transport`, + `supervise: external supervisor profile.harness=${JSON.stringify(canonicalProfile.harness)} requires a local bridge driverBackend, an explicit driveHarness, or resolveDriveHarness with reachable coordination transport`, ) } - const driveHarnessForOwner = (ownerId: string): DriveHarness | undefined => - options.driveHarness ?? - (managerBackend && automaticDriverBackendSupported(managerBackend) + const harnessClaims = new WeakMap< + DriveHarness, + { readonly owners: Set; steerable: boolean } + >() + const claimDriveHarness = (rawHarness: unknown, ownerId: string): DriveHarness => { + if (typeof rawHarness !== 'function') { + throw new ValidationError( + 'supervise: resolveDriveHarness must return a DriveHarness function', + ) + } + const harness = rawHarness as DriveHarness + const deliver: unknown = harness.deliver + if (deliver !== undefined && typeof deliver !== 'function') { + throw new ValidationError('supervise: driveHarness.deliver must be a function when provided') + } + const claim = harnessClaims.get(harness) + const conflictingOwner = claim + ? [...claim.owners].find((claimedOwner) => claimedOwner !== ownerId) + : undefined + const steerable = typeof deliver === 'function' + if (conflictingOwner !== undefined && (steerable || claim?.steerable === true)) { + throw new ValidationError( + `supervise: steerable driveHarness is already bound to manager owner ${JSON.stringify(conflictingOwner)}; resolveDriveHarness must return a distinct steerable instance for owner ${JSON.stringify(ownerId)}`, + ) + } + if (claim) { + claim.owners.add(ownerId) + claim.steerable ||= steerable + } else { + harnessClaims.set(harness, { owners: new Set([ownerId]), steerable }) + } + return harness + } + const driveHarnessForOwner = (context: DriveHarnessOwnerContext): DriveHarness | undefined => { + if (options.resolveDriveHarness) { + return claimDriveHarness(options.resolveDriveHarness(context), context.ownerId) + } + if (options.driveHarness) { + return claimDriveHarness(options.driveHarness, context.ownerId) + } + return managerBackend && automaticDriverBackendSupported(managerBackend) ? driveHarnessFromBackend( managerBackend, - externalExecutionId('supervised-manager', { runNamespace, ownerId }), + externalExecutionId('supervised-manager', { + runNamespace, + ownerId: context.ownerId, + }), options.now ?? Date.now, ) - : undefined) - const rootDriveHarness = driveHarnessForOwner(rootOwnerId) + : undefined + } + const rootDriveHarness = isExternalSupervisor(canonicalProfile) + ? driveHarnessForOwner( + freezeDetached({ + runId, + runNamespace, + ownerId: rootOwnerId, + depth: 0, + identity: rootExecution.identity, + profile: canonicalProfile, + task: canonicalTask, + }), + ) + : undefined const rootOwnerRuntime = !isExternalSupervisor(canonicalProfile) || rootDriveHarness === undefined ? undefined @@ -1061,6 +1180,19 @@ export function supervise(profile: SupervisorProfile, task: unknown, opts: Super ...spawnContext, ...(childExecution.ref ? { execution: childExecution.ref } : {}), }) + const postAuthorizationContext: AuthorizedSpawnContext = freezeDetached({ + profile: authorized, + parent, + parentIdentity, + execution: childExecution.identity, + parentNodeId: spawnContext.parentNodeId, + assignmentId: spawnContext.assignmentId, + task: spawnContext.task, + budget: spawnContext.budget, + label: spawnContext.label, + ...(spawnContext.key !== undefined ? { key: spawnContext.key } : {}), + depth, + }) const security = validateAgentProfileSecurity(authorized, securityPolicy) if (!security.ok) { const details = security.issues @@ -1070,12 +1202,30 @@ export function supervise(profile: SupervisorProfile, task: unknown, opts: Super throw new ValidationError(`supervise: spawned AgentProfile refused: ${details}`) } assertProfileModelsAllowed(authorized, options.allowedModels) - const driverInput = { profile: authorized, parent, depth } - const isDriver = options.isDriverProfile - ? options.isDriverProfile(driverInput) - : authorized.metadata?.role === 'driver' + let isDriver: boolean + if (options.isDriverProfile) { + const driverDecision: unknown = options.isDriverProfile(postAuthorizationContext) + if (typeof driverDecision !== 'boolean') { + throw new ValidationError('supervise: isDriverProfile must return a boolean') + } + isDriver = driverDecision + } else { + isDriver = authorized.metadata?.role === 'driver' + } if (!isDriver) { - return makeLeaf( + const selectedDeliverable = options.resolveDeliverable?.(postAuthorizationContext) + const leafDeliverable = + selectedDeliverable === undefined + ? options.deliverable + : captureDeliverable( + selectedDeliverable, + `supervise deliverable for ${JSON.stringify(spawnContext.label)}`, + ) + const makeSelectedLeaf = + leafDeliverable === options.deliverable + ? makeLeaf + : workerFromBackend(options.backend as ExecutorConfig, leafDeliverable) + return makeSelectedLeaf( authorized, Object.freeze({ ...authorizedContext, @@ -1093,10 +1243,23 @@ export function supervise(profile: SupervisorProfile, task: unknown, opts: Super spawnContext, depth, ) - const nestedDriveHarness = driveHarnessForOwner(ownerId) + const nestedDriveHarness = isExternalSupervisor(authorized) + ? driveHarnessForOwner( + freezeDetached({ + runId, + runNamespace, + ownerId, + depth, + identity: childExecution.identity, + assignmentId: spawnContext.assignmentId, + profile: authorized, + task: spawnContext.task, + }), + ) + : undefined if (isExternalSupervisor(authorized) && !nestedDriveHarness) { throw new ValidationError( - `supervise: authored external supervisor profile.harness=${JSON.stringify(authorized.harness)} requires a local bridge driverBackend or an explicit driveHarness with reachable coordination transport`, + `supervise: authored external supervisor profile.harness=${JSON.stringify(authorized.harness)} requires a local bridge driverBackend, an explicit driveHarness, or resolveDriveHarness with reachable coordination transport`, ) } assertProfileContract( @@ -1219,7 +1382,9 @@ export function supervise(profile: SupervisorProfile, task: unknown, opts: Super ...(options.compaction ? { compaction: options.compaction } : {}), }) - return createSupervisor().run(agent, canonicalTask, { + const supervisor = createSupervisor() + if (options.rootHandle) supervisor.attach(options.rootHandle) + return supervisor.run(agent, canonicalTask, { budget: options.budget, runId, journal, diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index 83ae4a44..4bb80454 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -34,6 +34,7 @@ import type { PriorCoordination } from './coordination-log' import { serveCoordinationMcp } from './coordination-mcp' import type { BusRecord } from './event-bus' import { bestDelivered, runFinalizer, runTree, type SupervisorFinalizer } from './finalizer' +import { createInbox } from './inbox' import { attestRuntimeOwnedScopeOwner, runtimeOwnedScopeOwnerRuntime } from './materialization' import { detachedSnapshot } from './snapshot' import type { StopRule } from './stop-rules' @@ -110,15 +111,27 @@ export type ObserveSupervisorNodeEvent = ( * seam the caller supplies (mirrors `makeWorkerAgent` for spawned children). It runs `profile` on * `task` in its backend (remote sandbox or local CLI bridge) with `coordinationMcpUrl` mounted as an MCP server, * so the harness calls spawn_agent / await_event / stop as native tools over the live scope. */ -export type DriveHarness = (args: { - readonly profile: SupervisorProfile - readonly task: unknown - readonly scope: Scope - readonly coordinationMcpUrl: string - /** Data-only product tool surface mounted on the coordination MCP. Runtime-owned drivers include - * this in their materialization evidence without persisting executable handlers. */ - readonly coordinationTools: ReadonlyArray> -}) => Promise +export interface DriveHarness { + (args: { + readonly profile: SupervisorProfile + readonly task: unknown + readonly scope: Scope + readonly coordinationMcpUrl: string + /** Data-only product tool surface mounted on the coordination MCP. Runtime-owned drivers include + * this in their materialization evidence without persisting executable handlers. */ + readonly coordinationTools: ReadonlyArray> + }): Promise + /** Optional live inbox for the manager session this adapter currently drives. Return `false` + * when no executor inbox is active instead of claiming a message was delivered. */ + deliver?(message: unknown): boolean +} + +/** Trusted manager identity available before its external harness starts. A product uses this to + * return one independently steerable harness session per recursive manager. */ +export type DriveHarnessOwnerContext = Omit + +/** Resolve an external harness for one exact Runtime-owned manager identity. */ +export type ResolveDriveHarness = (context: DriveHarnessOwnerContext) => DriveHarness export interface SupervisorAgentDeps { readonly blobs: ResultBlobStore @@ -238,6 +251,7 @@ export function supervisorAgent( // ROUTER arm: the in-process tool-loop. `routerBrain` is now an internal detail — the caller // passes a profile, not a hand-built brain (a test may still inject `deps.brain`). const brain = deps.brain ?? routerBrainFromProfile(stableProfile, deps) + const inbox = createInbox() const build = ( priorCoordination?: PriorCoordination, nodeTools?: ReadonlyArray, @@ -267,12 +281,16 @@ export function supervisorAgent( ...(deps.replaySettlements ? { replaySettlements: true } : {}), ...(priorCoordination ? { priorCoordination } : {}), ...(deps.finalizer ? { finalizer: deps.finalizer } : {}), + inbox, }) if (!deps.loadPriorCoordination && !resolveTools && !observeNodeEvent) { return build(deps.priorCoordination, undefined, deps.onEvent) } return { name, + deliver(message): boolean { + return inbox.deliver(message) + }, async act(task, scope) { const context = nodeContextSeed ? supervisorNodeContext(nodeContextSeed, stableProfile, task, scope) @@ -293,8 +311,16 @@ export function supervisorAgent( `supervisorAgent: profile.harness="${harness}" needs deps.driveHarness (how to run the harness with the coordination MCP mounted)`, ) } + const deliver = driveHarness.deliver?.bind(driveHarness) const externalAgent: Agent = { name, + ...(deliver + ? { + deliver(message: unknown): boolean { + return deliver(message) + }, + } + : {}), async act(task, scope) { const context = nodeContextSeed ? supervisorNodeContext(nodeContextSeed, stableProfile, task, scope) diff --git a/src/runtime/supervise/supervisor.ts b/src/runtime/supervise/supervisor.ts index 1ec142f0..82d9911a 100644 --- a/src/runtime/supervise/supervisor.ts +++ b/src/runtime/supervise/supervisor.ts @@ -67,6 +67,7 @@ import type { SpawnEvent, SpawnJournal, Spend, + SteerableRootHandle, SupervisedResult, Supervisor, SupervisorOpts, @@ -383,299 +384,313 @@ export function createSupervisor(): Supervisor { }) task = input.task const rootAct = root.act.bind(root) + const rootDeliver = root.deliver?.bind(root) const now = opts.now ?? Date.now assertRootIdentity(opts) - const rootAttemptId = newExecutionAttemptId(opts.runId) - // One instant owns both the fresh pool's absolute deadline and its durable root records. Capture - // it before any asynchronous journal work so a slow begin cannot extend the run on restart. - const runStartedAtMs = now() - const runStartedAt = new Date(runStartedAtMs).toISOString() + // Reserve the attached control synchronously, before the first journal read or write. A handle + // is one live route, so two concurrent runs may never race to overwrite that route and later + // detach each other. The lease is released on every early failure as well as normal teardown. + const rootLease = attached?.acquire() + try { + const rootAttemptId = newExecutionAttemptId(opts.runId) + // One instant owns both the fresh pool's absolute deadline and its durable root records. Capture + // it before any asynchronous journal work so a slow begin cannot extend the run on restart. + const runStartedAtMs = now() + const runStartedAt = new Date(runStartedAtMs).toISOString() - // RESUME-FIRST (opt-in via `opts.resume`): read any prior journal tree for this runId BEFORE - // beginning a fresh one. A non-empty tree means a prior run for this runId already committed - // work (it crashed mid-flight, or this is an explicit resume), so rehydrate it instead of - // starting over. An empty/absent tree — and every run that did NOT opt in — takes the fresh - // path unchanged. This wires the already-built+tested resume primitives - // (`replaySpawnTree`/`materializeTreeView`); the journal/blob store decide durability. - const existing = await opts.journal.loadTree(opts.runId) - if (opts.resume !== true && existing !== undefined) { - throw new RuntimeRunStateError( - `supervisor: runId '${opts.runId}' already exists; pass resume: true to continue it or use a new runId`, - ) - } - const prior = opts.resume === true ? existing : undefined - const resuming = prior !== undefined && prior.length > 0 - let resumeFrom: ResumeFrom | undefined - let pool: BudgetPool - if (resuming) { - const rootEvent = assertResumeContract(prior, opts) - const measured = sumMeasuredSpendFromEvents(prior) - const uncertainReservations = uncertainSpawnBudgets(prior) - pool = createBudgetPool(opts.budget, now, { - committed: addSpend(measured.childWork, measured.driverInference), - uncertainReservations, - ...(rootEvent.budget.deadlineMs !== undefined - ? { absoluteDeadlineMs: rootDeadline(rootEvent) } - : {}), - }) - // Rehydrate the committed work: the cursor-ordered `Settled[]` (from the blob store) plus the - // tree as it stood at the recorded cursor position. The new scope's ordinal/cursor counters - // continue past the recorded maxima so a fresh spawn never reuses a journaled `seq`. - const settled = await replaySpawnTree(opts.journal, opts.blobs, opts.runId) - const view = materializeTreeView(prior) - resumeFrom = { - settled, - view, - maxSpawnOrdinal: maxSeqOf(prior, (ev) => ev.kind === 'spawned'), - maxCursorSeq: maxSeqOf( - prior, - (ev) => ev.kind === 'settled' || ev.kind === 'cancelled' || ev.kind === 'woken', - ), - maxWaitOrdinal: maxSeqOf(prior, (ev) => ev.kind === 'waiting'), - // Waits armed but never woken: the run died mid-wait. They ride onto `Scope.resume.waits`, - // and re-arming the same label adopts the ORIGINAL absolute deadline rather than - // restarting the countdown from this process's clock. - waits: pendingWaits(prior), - // Keyed assignments + prior committed spend ride onto `Scope.resume`, so a resume-aware - // driver resolves keys instead of re-spawning and reports what the run already paid. - keys: keyedAssignments(prior, settled), - priorSpend: sumSpendFromEvents(prior), + // RESUME-FIRST (opt-in via `opts.resume`): read any prior journal tree for this runId BEFORE + // beginning a fresh one. A non-empty tree means a prior run for this runId already committed + // work (it crashed mid-flight, or this is an explicit resume), so rehydrate it instead of + // starting over. An empty/absent tree — and every run that did NOT opt in — takes the fresh + // path unchanged. This wires the already-built+tested resume primitives + // (`replaySpawnTree`/`materializeTreeView`); the journal/blob store decide durability. + const existing = await opts.journal.loadTree(opts.runId) + if (opts.resume !== true && existing !== undefined) { + throw new RuntimeRunStateError( + `supervisor: runId '${opts.runId}' already exists; pass resume: true to continue it or use a new runId`, + ) } - } else { - pool = createBudgetPool(opts.budget, now, { - ...(opts.budget.deadlineMs !== undefined - ? { absoluteDeadlineMs: runStartedAtMs + opts.budget.deadlineMs } - : {}), - }) - // Fresh run: begin the tree and journal the root as its own `spawned` node (parent-less, the - // spawn-ordinal-0 marker), so a journal-based reader — `trajectoryReport`, `replaySpawnTree`, - // `materializeTreeView` — can reconstruct the WHOLE realized tree from a real run, not only - // hand-built journals. The root is never `scope.spawn`ed (the supervisor runs `act` directly), - // so without this the root node is absent and `trajectoryReport` fails its `nodes.has(root)` - // invariant. The uniqueness guard skips `spawned` events (only the cursor namespace must be - // unique), so sharing ordinal 0 with the first child's spawn is not a collision; replay ignores - // `spawned` events for settlement reconstruction, so the replayed `Settled[]` is unchanged. - await opts.journal.beginTree(opts.runId, runStartedAt) - const rootReceipt = rootMaterializationReceipt(opts) - const rootRuntime = opts.rootMaterialization?.runtime ?? 'inline' - await opts.journal.appendEvent(opts.runId, { - kind: 'spawned', - id: opts.runId, - label: 'root', - budget: opts.budget, - runtime: rootRuntime, - ...(opts.rootIdentity ? { identity: opts.rootIdentity } : {}), - seq: 0, - at: runStartedAt, - }) - if (rootReceipt !== undefined) { + const prior = opts.resume === true ? existing : undefined + const resuming = prior !== undefined && prior.length > 0 + let resumeFrom: ResumeFrom | undefined + let pool: BudgetPool + if (resuming) { + const rootEvent = assertResumeContract(prior, opts) + const measured = sumMeasuredSpendFromEvents(prior) + const uncertainReservations = uncertainSpawnBudgets(prior) + pool = createBudgetPool(opts.budget, now, { + committed: addSpend(measured.childWork, measured.driverInference), + uncertainReservations, + ...(rootEvent.budget.deadlineMs !== undefined + ? { absoluteDeadlineMs: rootDeadline(rootEvent) } + : {}), + }) + // Rehydrate the committed work: the cursor-ordered `Settled[]` (from the blob store) plus the + // tree as it stood at the recorded cursor position. The new scope's ordinal/cursor counters + // continue past the recorded maxima so a fresh spawn never reuses a journaled `seq`. + const settled = await replaySpawnTree(opts.journal, opts.blobs, opts.runId) + const view = materializeTreeView(prior) + resumeFrom = { + settled, + view, + maxSpawnOrdinal: maxSeqOf(prior, (ev) => ev.kind === 'spawned'), + maxCursorSeq: maxSeqOf( + prior, + (ev) => ev.kind === 'settled' || ev.kind === 'cancelled' || ev.kind === 'woken', + ), + maxWaitOrdinal: maxSeqOf(prior, (ev) => ev.kind === 'waiting'), + // Waits armed but never woken: the run died mid-wait. They ride onto `Scope.resume.waits`, + // and re-arming the same label adopts the ORIGINAL absolute deadline rather than + // restarting the countdown from this process's clock. + waits: pendingWaits(prior), + // Keyed assignments + prior committed spend ride onto `Scope.resume`, so a resume-aware + // driver resolves keys instead of re-spawning and reports what the run already paid. + keys: keyedAssignments(prior, settled), + priorSpend: sumSpendFromEvents(prior), + } + } else { + pool = createBudgetPool(opts.budget, now, { + ...(opts.budget.deadlineMs !== undefined + ? { absoluteDeadlineMs: runStartedAtMs + opts.budget.deadlineMs } + : {}), + }) + // Fresh run: begin the tree and journal the root as its own `spawned` node (parent-less, the + // spawn-ordinal-0 marker), so a journal-based reader — `trajectoryReport`, `replaySpawnTree`, + // `materializeTreeView` — can reconstruct the WHOLE realized tree from a real run, not only + // hand-built journals. The root is never `scope.spawn`ed (the supervisor runs `act` directly), + // so without this the root node is absent and `trajectoryReport` fails its `nodes.has(root)` + // invariant. The uniqueness guard skips `spawned` events (only the cursor namespace must be + // unique), so sharing ordinal 0 with the first child's spawn is not a collision; replay ignores + // `spawned` events for settlement reconstruction, so the replayed `Settled[]` is unchanged. + await opts.journal.beginTree(opts.runId, runStartedAt) + const rootReceipt = rootMaterializationReceipt(opts) + const rootRuntime = opts.rootMaterialization?.runtime ?? 'inline' await opts.journal.appendEvent(opts.runId, { - kind: 'materialized', + kind: 'spawned', id: opts.runId, - receipt: rootReceipt, + label: 'root', + budget: opts.budget, + runtime: rootRuntime, + ...(opts.rootIdentity ? { identity: opts.rootIdentity } : {}), seq: 0, at: runStartedAt, }) + if (rootReceipt !== undefined) { + await opts.journal.appendEvent(opts.runId, { + kind: 'materialized', + id: opts.runId, + receipt: rootReceipt, + seq: 0, + at: runStartedAt, + }) + } } - } - const stableRootReceipt = resuming - ? prior?.find( - (event): event is Extract => - event.kind === 'materialized' && event.id === opts.runId, - )?.receipt - : rootMaterializationReceipt(opts) - if (stableRootReceipt !== undefined) { - const rootBinding = rootExecutionBindingReceipt(opts, stableRootReceipt, rootAttemptId) - if (rootBinding !== undefined) { - await opts.journal.appendEvent(opts.runId, { - kind: 'execution-bound', - id: opts.runId, - binding: rootBinding, - seq: 0, - at: runStartedAt, - }) + const stableRootReceipt = resuming + ? prior?.find( + (event): event is Extract => + event.kind === 'materialized' && event.id === opts.runId, + )?.receipt + : rootMaterializationReceipt(opts) + if (stableRootReceipt !== undefined) { + const rootBinding = rootExecutionBindingReceipt(opts, stableRootReceipt, rootAttemptId) + if (rootBinding !== undefined) { + await opts.journal.appendEvent(opts.runId, { + kind: 'execution-bound', + id: opts.runId, + binding: rootBinding, + seq: 0, + at: runStartedAt, + }) + } } - } - // ONE internal controller is the root scope's abort source. Every cascade path - // (caller signal, RootHandle.abort, breaker trip, deadline) aborts it; the scope - // fans it out to each live child's executor (acquire-aware reap included). - const controller = new AbortController() - const cascadeAbort = (reason?: string): boolean => { - if (controller.signal.aborted) return false - // Carry the reason on the signal so it chains down to each child's abort signal - // (`childAbort.signal.reason`) — the diagnostic the scope's executors observe. - controller.abort(reason) - return true - } + // ONE internal controller is the root scope's abort source. Every cascade path + // (caller signal, RootHandle.abort, breaker trip, deadline) aborts it; the scope + // fans it out to each live child's executor (acquire-aware reap included). + const controller = new AbortController() + const cascadeAbort = (reason?: string): boolean => { + if (controller.signal.aborted) return false + // Carry the reason on the signal so it chains down to each child's abort signal + // (`childAbort.signal.reason`) — the diagnostic the scope's executors observe. + controller.abort(reason) + return true + } - const onCallerAbort = () => cascadeAbort('caller signal aborted') - if (opts.signal) { - if (opts.signal.aborted) cascadeAbort('caller signal aborted') - else opts.signal.addEventListener('abort', onCallerAbort, { once: true }) - } + const onCallerAbort = () => cascadeAbort('caller signal aborted') + if (opts.signal) { + if (opts.signal.aborted) cascadeAbort('caller signal aborted') + else opts.signal.addEventListener('abort', onCallerAbort, { once: true }) + } - // The breaker watches `down` settlements via a counting journal decorator, so it - // observes every child failure without intercepting `scope.next()` (the driver's - // private channel). Tripping aborts the same controller; the trip is recorded so the - // final result can name it. - const breaker = createIntensityBreaker(opts, () => cascadeAbort('intensity breaker tripped')) - const journal = wrapJournalForBreaker(opts.journal, breaker) - const priorRootMaterialization = prior?.find( - (event): event is Extract => - event.kind === 'materialized' && event.id === opts.runId, - )?.receipt + // The breaker watches `down` settlements via a counting journal decorator, so it + // observes every child failure without intercepting `scope.next()` (the driver's + // private channel). Tripping aborts the same controller; the trip is recorded so the + // final result can name it. + const breaker = createIntensityBreaker(opts, () => cascadeAbort('intensity breaker tripped')) + const journal = wrapJournalForBreaker(opts.journal, breaker) + const priorRootMaterialization = prior?.find( + (event): event is Extract => + event.kind === 'materialized' && event.id === opts.runId, + )?.receipt - const scope = createScope({ - parentId: opts.runId, - root: opts.runId, - pool, - journal, - blobs: opts.blobs, - executors: opts.executors, - seams: {}, - depth: 0, - maxDepth: opts.maxDepth ?? defaultMaxDepth, - ...(opts.maxLiveWorkers !== undefined ? { maxLiveWorkers: opts.maxLiveWorkers } : {}), - signal: controller.signal, - now, - hooks: opts.hooks, - ...(opts.rootMaterialization?.declaration === 'deferred' - ? { - ownerMaterialization: { - runtime: opts.rootMaterialization.runtime, - authoredProfile: opts.rootMaterialization.authoredProfile, - attemptId: rootAttemptId, - requiredKnown: true, - ...(priorRootMaterialization === undefined - ? {} - : { prior: priorRootMaterialization }), - }, - } - : {}), - ...(opts.probes ? { probes: opts.probes } : {}), - ...(resumeFrom ? { resumeFrom } : {}), - }) + const scope = createScope({ + parentId: opts.runId, + root: opts.runId, + pool, + journal, + blobs: opts.blobs, + executors: opts.executors, + seams: {}, + depth: 0, + maxDepth: opts.maxDepth ?? defaultMaxDepth, + ...(opts.maxLiveWorkers !== undefined ? { maxLiveWorkers: opts.maxLiveWorkers } : {}), + signal: controller.signal, + now, + hooks: opts.hooks, + ...(opts.rootMaterialization?.declaration === 'deferred' + ? { + ownerMaterialization: { + runtime: opts.rootMaterialization.runtime, + authoredProfile: opts.rootMaterialization.authoredProfile, + attemptId: rootAttemptId, + requiredKnown: true, + ...(priorRootMaterialization === undefined + ? {} + : { prior: priorRootMaterialization }), + }, + } + : {}), + ...(opts.probes ? { probes: opts.probes } : {}), + ...(resumeFrom ? { resumeFrom } : {}), + }) - // `view`/drain read the scope opaquely (`Out` erased) — the supervisor never `spawn`s - // on it, so the live-tree readout and the join barrier are `Out`-agnostic. - const openScope = scope as unknown as Scope + // `view`/drain read the scope opaquely (`Out` erased) — the supervisor never `spawn`s + // on it, so the live-tree readout and the join barrier are `Out`-agnostic. + const openScope = scope as unknown as Scope - // Bind any attached RootHandle to THIS live run so view()/signal()/abort() reach the - // live scope + the one cascade controller. Detached again in the finally barrier. - if (attached) { - attached.bind({ scope: openScope, cascadeAbort, signal: pushRootSignal(cascadeAbort) }) - } + // Bind any attached RootHandle to THIS live run so view()/signal()/abort() reach the + // live scope + the one cascade controller. Detached again in the finally barrier. + if (rootLease) { + rootLease.bind({ + scope: openScope, + cascadeAbort, + signal: pushRootSignal(cascadeAbort), + deliver: rootDeliver ? (message) => rootDeliver(message) !== false : () => false, + }) + } - let deadlineExceeded = false - const rootDeadlineAtMs = pool.readout().deadlineMs - if (rootDeadlineAtMs > 0 && now() >= rootDeadlineAtMs) { - deadlineExceeded = cascadeAbort('root budget deadline exceeded') - } - const clearRootDeadline = - opts.budget.deadlineMs === undefined - ? undefined - : armDeadlineTimer(Math.max(0, rootDeadlineAtMs - now()), () => { - deadlineExceeded = cascadeAbort('root budget deadline exceeded') - }) - let actOutcome: { ok: true; out: Out } | { ok: false; error: unknown } = { - ok: false, - error: new RuntimeRunStateError('supervisor: root execution did not start'), - } - let executionAborted = controller.signal.aborted - try { - const out = await runAbortable(() => rootAct(task, scope), controller.signal) - actOutcome = { ok: true, out } - } catch (error) { - // act()'s rejection is the PRIMARY error; capture it before the join barrier so a - // teardown failure in the barrier can never overwrite it (firstError precedence). - actOutcome = { ok: false, error } - } finally { - executionAborted = controller.signal.aborted - // A child inherits the root cutoff and can settle the root act on that exact timer turn. - // If its settlement callback runs before the root timer callback, the finally block clears - // the still-pending root timer. Preserve the deadline cause from the shared absolute clock; - // do not overwrite a caller/breaker abort that already won the controller race. - if (!controller.signal.aborted && rootDeadlineAtMs > 0 && now() >= rootDeadlineAtMs) { - deadlineExceeded = true + let deadlineExceeded = false + const rootDeadlineAtMs = pool.readout().deadlineMs + if (rootDeadlineAtMs > 0 && now() >= rootDeadlineAtMs) { + deadlineExceeded = cascadeAbort('root budget deadline exceeded') } - clearRootDeadline?.() - // Join barrier: tear down every still-live child. Generalizes the kernel's - // `finally{ Promise.allSettled(destroy) }` — a teardown throw is allSettled'd and - // journaled, never re-thrown. - try { - await drainLiveChildren(openScope, controller) - } catch (error) { - if (actOutcome?.ok !== false) actOutcome = { ok: false, error } + const clearRootDeadline = + opts.budget.deadlineMs === undefined + ? undefined + : armDeadlineTimer(Math.max(0, rootDeadlineAtMs - now()), () => { + deadlineExceeded = cascadeAbort('root budget deadline exceeded') + }) + let actOutcome: { ok: true; out: Out } | { ok: false; error: unknown } = { + ok: false, + error: new RuntimeRunStateError('supervisor: root execution did not start'), } + let executionAborted = controller.signal.aborted try { - await finalizeScopeOwnerMaterialization(openScope) + const out = await runAbortable(() => rootAct(task, scope), controller.signal) + actOutcome = { ok: true, out } } catch (error) { - if (actOutcome?.ok !== false) actOutcome = { ok: false, error } + // act()'s rejection is the PRIMARY error; capture it before the join barrier so a + // teardown failure in the barrier can never overwrite it (firstError precedence). + actOutcome = { ok: false, error } + } finally { + executionAborted = controller.signal.aborted + // A child inherits the root cutoff and can settle the root act on that exact timer turn. + // If its settlement callback runs before the root timer callback, the finally block clears + // the still-pending root timer. Preserve the deadline cause from the shared absolute clock; + // do not overwrite a caller/breaker abort that already won the controller race. + if (!controller.signal.aborted && rootDeadlineAtMs > 0 && now() >= rootDeadlineAtMs) { + deadlineExceeded = true + } + clearRootDeadline?.() + // Join barrier: tear down every still-live child. Generalizes the kernel's + // `finally{ Promise.allSettled(destroy) }` — a teardown throw is allSettled'd and + // journaled, never re-thrown. + try { + await drainLiveChildren(openScope, controller) + } catch (error) { + if (actOutcome?.ok !== false) actOutcome = { ok: false, error } + } + try { + await finalizeScopeOwnerMaterialization(openScope) + } catch (error) { + if (actOutcome?.ok !== false) actOutcome = { ok: false, error } + } + if (opts.signal) opts.signal.removeEventListener('abort', onCallerAbort) + rootLease?.release() } - if (opts.signal) opts.signal.removeEventListener('abort', onCallerAbort) - if (attached) attached.unbind() - } - // The run's tree, not this process's: on a resumed run the prior process's committed nodes are - // carried in, so `tree` covers the same work `spentTotal` bills for. Identical to `scope.view` - // on every run that did not resume. - const tree = runTree(scope) - // Success and failure both pass the same conservation check. A child promise is never allowed - // to disappear behind a swallowed cleanup error and leave a reservation open. - pool.assertNoOpenTickets() - if (actOutcome.ok) { - if (executionAborted) return noWinner() - // Every child has settled (join barrier above); no reservation may remain. A leaked ticket - // would silently corrupt the conserved spend total, so fail loud here — on the success path - // only, where the act() error precedence does not apply. - const out = actOutcome.out - // Completion-oracle at the root: a `winner` MUST carry a real `Out`. A driver that ran to - // completion but selected nothing (its keep-best finalize found no DELIVERED child) returns - // `undefined` — that is a no-winner, never a winner wrapping `undefined`. The supervisor's - // contract is to refuse coercing a non-result into a best-effort Out (Foreman's 0/18 lesson). - if (out !== undefined) { - // The driver synthesized a winner. Content-address it for the replay `outRef`, put it - // once, and sum the conserved spend off every journaled settlement. No re-ranking — the - // driver already selected. - const outRef = contentAddress(out) - await opts.blobs.put(outRef, out) - // ONE ledger: the journal. `settled` events carry spawned-child WORK; `metered` events carry - // the drivers' OWN inference (the twin of `pool.observe`). `spentTotal` is their sum and the - // breakdown keeps the two separable — the A++ view of where the tokens went. No pool bridge. + // The run's tree, not this process's: on a resumed run the prior process's committed nodes are + // carried in, so `tree` covers the same work `spentTotal` bills for. Identical to `scope.view` + // on every run that did not resume. + const tree = runTree(scope) + // Success and failure both pass the same conservation check. A child promise is never allowed + // to disappear behind a swallowed cleanup error and leave a reservation open. + pool.assertNoOpenTickets() + if (actOutcome.ok) { + if (executionAborted) return noWinner() + // Every child has settled (join barrier above); no reservation may remain. A leaked ticket + // would silently corrupt the conserved spend total, so fail loud here — on the success path + // only, where the act() error precedence does not apply. + const out = actOutcome.out + // Completion-oracle at the root: a `winner` MUST carry a real `Out`. A driver that ran to + // completion but selected nothing (its keep-best finalize found no DELIVERED child) returns + // `undefined` — that is a no-winner, never a winner wrapping `undefined`. The supervisor's + // contract is to refuse coercing a non-result into a best-effort Out (Foreman's 0/18 lesson). + if (out !== undefined) { + // The driver synthesized a winner. Content-address it for the replay `outRef`, put it + // once, and sum the conserved spend off every journaled settlement. No re-ranking — the + // driver already selected. + const outRef = contentAddress(out) + await opts.blobs.put(outRef, out) + // ONE ledger: the journal. `settled` events carry spawned-child WORK; `metered` events carry + // the drivers' OWN inference (the twin of `pool.observe`). `spentTotal` is their sum and the + // breakdown keeps the two separable — the A++ view of where the tokens went. No pool bridge. + const { childWork, driverInference } = await spentFromJournal(journal, opts.runId) + return { + kind: 'winner', + out, + outRef, + tree, + spentTotal: addSpend(childWork, driverInference), + ...(isNonEmptySpend(driverInference) + ? { spentBreakdown: { driverInference, childWork } } + : {}), + } + } + return noWinner() + } + + // act() rejected. The reason is proven from lifecycle state, in precedence order: + // a tripped breaker outranks any abort (it is the most specific cause) outranks + // budget-exhaustion outranks the residual "the tree produced nothing usable" bucket. + // A no-winner is TYPED — never a best-effort coercion of a partial child (M2). + return noWinner() + + // A no-winner still incurred real conserved spend before failing, so it carries `spentTotal` + // summed off the SAME journal the winner path reads — the caller always learns the cost. + async function noWinner(): Promise> { const { childWork, driverInference } = await spentFromJournal(journal, opts.runId) return { - kind: 'winner', - out, - outRef, + kind: 'no-winner', + reason: classifyNoWinner(controller, pool, opts, breaker, deadlineExceeded, tree), tree, + downCount: breaker.downCount(), spentTotal: addSpend(childWork, driverInference), - ...(isNonEmptySpend(driverInference) - ? { spentBreakdown: { driverInference, childWork } } - : {}), } } - return noWinner() - } - - // act() rejected. The reason is proven from lifecycle state, in precedence order: - // a tripped breaker outranks any abort (it is the most specific cause) outranks - // budget-exhaustion outranks the residual "the tree produced nothing usable" bucket. - // A no-winner is TYPED — never a best-effort coercion of a partial child (M2). - return noWinner() - - // A no-winner still incurred real conserved spend before failing, so it carries `spentTotal` - // summed off the SAME journal the winner path reads — the caller always learns the cost. - async function noWinner(): Promise> { - const { childWork, driverInference } = await spentFromJournal(journal, opts.runId) - return { - kind: 'no-winner', - reason: classifyNoWinner(controller, pool, opts, breaker, deadlineExceeded, tree), - tree, - downCount: breaker.downCount(), - spentTotal: addSpend(childWork, driverInference), - } + } finally { + rootLease?.release() } } @@ -700,13 +715,18 @@ interface RunBinding { readonly scope: Scope readonly cascadeAbort: (reason?: string) => void readonly signal: (msg: RootSignal) => void + readonly deliver: (msg: unknown) => boolean } /** The supervisor-private control behind a `RootHandle`. `createRootHandle` mints it and * registers it in `rootControls`; `attach` looks it up and `bind`s it to the live run. */ -interface RootControl { +interface RootLease { bind(binding: RunBinding): void - unbind(): void + release(): void +} + +interface RootControl { + acquire(): RootLease } /** Module-private channel from a minted `RootHandle` to its `RootControl`, so `attach` @@ -721,9 +741,10 @@ const rootControls = new WeakMap, RootControl>() * unbinds it) the handle is fail-loud: a client that talks to a handle that is not * driving a live run gets a typed error, never a silent no-op. */ -export function createRootHandle(): RootHandle { +export function createRootHandle(): SteerableRootHandle { let binding: RunBinding | undefined - const handle: RootHandle = { + let activeLease: symbol | undefined + const handle: SteerableRootHandle = { view(): TreeView { if (!binding) { throw new RuntimeRunStateError( @@ -732,6 +753,12 @@ export function createRootHandle(): RootHandle { } return binding.scope.view }, + deliver(msg: unknown): boolean { + if (!binding) { + throw new RuntimeRunStateError('RootHandle.deliver: handle is not bound to a live run') + } + return binding.deliver(msg) + }, signal(msg: RootSignal): void { if (!binding) { throw new RuntimeRunStateError('RootHandle.signal: handle is not bound to a live run') @@ -746,11 +773,33 @@ export function createRootHandle(): RootHandle { }, } rootControls.set(handle as RootHandle, { - bind(b: RunBinding): void { - binding = b - }, - unbind(): void { - binding = undefined + acquire(): RootLease { + if (activeLease !== undefined) { + throw new RuntimeRunStateError( + 'RootHandle: handle already controls a live run; use one handle per concurrent run', + ) + } + const token = Symbol('root-handle-lease') + activeLease = token + let released = false + return { + bind(next: RunBinding): void { + if (released || activeLease !== token) { + throw new RuntimeRunStateError('RootHandle: live-run lease is no longer active') + } + if (binding !== undefined) { + throw new RuntimeRunStateError('RootHandle: live-run lease is already bound') + } + binding = next + }, + release(): void { + if (released) return + released = true + if (activeLease !== token) return + binding = undefined + activeLease = undefined + }, + } }, }) return handle diff --git a/src/runtime/supervise/tree-key.ts b/src/runtime/supervise/tree-key.ts new file mode 100644 index 00000000..3fe3b3bf --- /dev/null +++ b/src/runtime/supervise/tree-key.ts @@ -0,0 +1,31 @@ +import type { NodeId } from './types' + +/** Runtime tag used only for a child that owns another supervised scope. */ +export const driverRuntime = 'driver' as const + +// Recursive ownership is an executor capability, not a Runtime string. `Runtime` is deliberately +// open, so a caller-owned leaf may also call itself "driver". Keep the capability on the exact +// Runtime-created executor object and never expose a public lookalike property that a leaf can set. +const nestedDriverTreeOwners = new WeakSet() + +/** Canonical journal key for the scope owned by one nested driver node. */ +export function nestedDriverTreeRoot(parentTreeRoot: NodeId, driverNodeId: NodeId): NodeId { + return `${parentTreeRoot}/${driverNodeId}` +} + +/** Privately mark the exact recursive executor that will create a nested journal tree. */ +export function attestNestedDriverTreeOwner(owner: T): T { + nestedDriverTreeOwners.add(owner) + return owner +} + +/** Return the canonical owned tree only for an exact privately marked recursive executor. */ +export function runtimeOwnedNestedDriverTreeRoot( + owner: object, + parentTreeRoot: NodeId, + driverNodeId: NodeId, +): NodeId | undefined { + return nestedDriverTreeOwners.has(owner) + ? nestedDriverTreeRoot(parentTreeRoot, driverNodeId) + : undefined +} diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 604e0923..576a553d 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -69,6 +69,13 @@ export interface WaitOpts { export interface Agent { readonly name: string act(task: Task, scope: Scope): Promise + /** + * Optional manager inbox. A parent or attached `RootHandle` uses this to deliver the same raw + * down-message accepted by executor inboxes. Return `false` when the manager has no live receive + * path; returning `true` means the message was accepted for the current manager session. + */ + // biome-ignore lint/suspicious/noConfusingVoidType: void is the legacy contract; boolean adds an acknowledgement without breaking existing agents. + deliver?(msg: unknown): void | boolean } // ── The open leaf runtime ───────────────────────────────────────────────────── @@ -110,10 +117,11 @@ export interface Executor { * Optional inbox: receive an out-of-band message from the driver mid-run (the `send`/`steer_agent` * verb). A streaming executor drains pending messages between turns and folds them into the next * step (a steer / interrupt / resume). A one-shot executor that can't be steered mid-flight omits - * this; `Scope.send` then returns `false` for it. Never throws — a malformed message is the - * executor's to ignore. + * this; `Scope.send` then returns `false` for it. Never throws — an inbox that rejects a malformed + * message returns `false`, and that refusal propagates to the caller. */ - deliver?(msg: unknown): void + // biome-ignore lint/suspicious/noConfusingVoidType: void is the legacy contract; boolean adds an acknowledgement without breaking existing executors. + deliver?(msg: unknown): void | boolean /** * Optional LIVE progress: what this worker is doing RIGHT NOW, read synchronously and * cheaply while `execute` is still streaming. The scope already derives activity timing, @@ -746,6 +754,8 @@ export interface NodeSnapshot { readonly status: NodeStatus readonly runtime: Runtime readonly budget: Budget + /** Exact nested journal tree owned by this node, when Runtime attested recursive ownership. */ + readonly ownedTreeRoot?: NodeId /** Manager-scoped assignment identity, including deterministic ids for unkeyed siblings. */ readonly assignmentId?: string readonly identity?: NodeExecutionIdentity @@ -792,6 +802,10 @@ export type SpawnEvent = assignmentId?: string budget: Budget runtime: Runtime + /** Exact nested journal tree this node owns. Runtime writes this only after privately + * attesting the executor as a recursive scope owner. Its absence means no tree is followed, + * including records written before this field existed and caller leaves named `driver`. */ + ownedTreeRoot?: NodeId /** Exact profile/task digests plus trusted candidate/campaign attribution when available. */ identity?: NodeExecutionIdentity seq: number @@ -978,10 +992,12 @@ export type SupervisedResult = spentTotal: Spend } -/** Live root handle — the substrate a chat/pi-viz client attaches to (Q2). `signal` - * delivers an out-of-band message to the running root; `view()` materializes the tree. */ +/** Live root handle — a chat/pi-viz client uses it to inspect and control one root run. */ export interface RootHandle { view(): TreeView + /** Optional for structural compatibility with existing view/signal/abort wrappers. Handles + * minted by `createRootHandle` implement the required form in `SteerableRootHandle`. */ + deliver?(msg: unknown): boolean signal(msg: RootSignal): void abort(reason?: string): void /** Phantom: binds the handle to the supervised run's output type. Type-only — never @@ -989,6 +1005,12 @@ export interface RootHandle { readonly __out?: Out } +/** A Runtime-minted root handle that can deliver raw steering or answers to a live manager inbox. + * Delivery returns `false` when the manager has no receive path; detached calls fail loud. */ +export interface SteerableRootHandle extends RootHandle { + deliver(msg: unknown): boolean +} + /** Out-of-band message to a running root. Open by intent — a client extends it. */ export type RootSignal = | { kind: 'pause' } diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 86788a8b..3dd829a3 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:fc52fd656a37a79611a2f870079695fe1a5c055d3078d58b0f2d4c79d73c3945", + "digest": "sha256:915091eb7a6e7fa350809a2c0d26bde069d5f36d2530e940ef3f3aa4d4c8393b", "evaluation": { "decision": { "contributingChecks": [ @@ -4810,7 +4810,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.109.2" + "runtimeVersion": "0.110.0" }, "objectives": [ { @@ -4921,8 +4921,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:042b806bcb6e3db2fe3b8744d543d769a0d3edf6cebe5413becb137196b00fd8", - "runId": "agent-runtime-0.109.2-proposal-fixture", + "recordDigest": "sha256:91fb7ed1a8eb813cb23bc03102aa98d0ba2eb5414a1956ccd756b85a0885ce40", + "runId": "agent-runtime-0.110.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -4949,5 +4949,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.109.2-proposal-fixture" + "runId": "agent-runtime-0.110.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 61f9ef3d..de103761 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:5a6888221983315e661a139d32348538dc054443e03ce2ef7f0cdf6380fb3558", + "digest": "sha256:8682eefa956d465e0b9cbcbf0783fddf7570cf94ec209aab5604eed69f9e63fc", "evaluation": { "decision": { "contributingChecks": [ @@ -1695,7 +1695,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.109.2" + "runtimeVersion": "0.110.0" }, "objectives": [ { diff --git a/tests/candidate-execution-export-surface.test.ts b/tests/candidate-execution-export-surface.test.ts new file mode 100644 index 00000000..400b1649 --- /dev/null +++ b/tests/candidate-execution-export-surface.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest' +import { + freezeGenericAgentCandidateProfile as fromCandidateExecution, + omitUndefinedObjectFields as omitFromCandidateExecution, + parseExactCandidateProfile as parseFromCandidateExecution, +} from '../src/candidate-execution' +import { + freezeGenericAgentCandidateProfile as fromRoot, + omitUndefinedObjectFields as omitFromRoot, + parseExactCandidateProfile as parseFromRoot, +} from '../src/index' + +describe('candidate profile conversion public surface', () => { + it('exports the canonical conversion and exact parser from both public barrels', () => { + expect(fromRoot).toBe(fromCandidateExecution) + expect(parseFromRoot).toBe(parseFromCandidateExecution) + expect(omitFromRoot).toBe(omitFromCandidateExecution) + + const candidate = fromRoot({ + name: 'public-seed', + prompt: { systemPrompt: 'Lead the pursuit.' }, + }) + expect(parseFromRoot(candidate)).toEqual(candidate) + expect(omitFromRoot({ keep: 1, drop: undefined }, 'profile')).toEqual({ keep: 1 }) + }) +}) diff --git a/tests/kernel/driver-recursion.test.ts b/tests/kernel/driver-recursion.test.ts index 81a3fae2..73a44059 100644 --- a/tests/kernel/driver-recursion.test.ts +++ b/tests/kernel/driver-recursion.test.ts @@ -144,6 +144,58 @@ function supervisorOpts(over: Partial = {}): SupervisorOpts { } describe('recursive driver: agents drive agents drive agents', () => { + it('delivers a parent steer through driverChild and the nested driver executor', async () => { + const journal = new InMemorySpawnJournal() + const blobs = new InMemoryResultBlobStore() + let received: unknown + let acceptMessage!: () => void + const message = new Promise((resolve) => { + acceptMessage = resolve + }) + const nested: Agent = { + name: 'nested-manager', + deliver(value): boolean { + received = value + acceptMessage() + return true + }, + async act() { + await message + return received + }, + } + const root: Agent = { + name: 'root', + async act(task, scope) { + const spawned = scope.spawn(driverChild('nested-manager', nested, journal), task, { + budget: perChild, + label: 'nested-manager', + }) + if (!spawned.ok) throw new Error(spawned.reason) + expect( + scope.send(spawned.handle.id, { + steer: 'change the experiment before continuing', + interrupt: true, + }), + ).toBe(true) + await scope.next() + return received + }, + } + + const result = await createSupervisor().run( + root, + 'task', + supervisorOpts({ runId: 'nested-delivery', journal, blobs }), + ) + + expect(result.kind).toBe('winner') + expect(received).toEqual({ + steer: 'change the experiment before continuing', + interrupt: true, + }) + }) + it('a driver spawns a driver spawns a worker (depth-2 tree settles, root selects)', async () => { const journal = new InMemorySpawnJournal() const blobs = new InMemoryResultBlobStore() @@ -196,6 +248,57 @@ describe('recursive driver: agents drive agents drive agents', () => { expect(observed.settledIds).toContain('rec:s0:s0') // worker settled into the nested scope }) + it('keeps a nested manager invalid when its finalizer refuses a valid child', async () => { + const journal = new InMemorySpawnJournal() + const blobs = new InMemoryResultBlobStore() + const worker = workerLeaf('worker', { + out: { partial: true }, + tokens: { input: 5, output: 5 }, + iterations: 1, + score: 1, + }) + const refusingManager: Agent = { + name: 'refusing-manager', + async act(task, scope) { + const spawned = scope.spawn(worker, task, { + budget: perChild, + label: 'worker', + }) + if (!spawned.ok) throw new Error(spawned.reason) + await scope.next() + return undefined + }, + } + const root: Agent = { + name: 'root', + async act(task, scope) { + const spawned = scope.spawn( + driverChild('refusing-manager', refusingManager, journal), + task, + { budget: perChild, label: 'refusing-manager' }, + ) + if (!spawned.ok) throw new Error(spawned.reason) + await scope.next() + return undefined + }, + } + + const result = await createSupervisor().run( + root, + 'task', + supervisorOpts({ runId: 'finalizer-refusal', journal, blobs }), + ) + + expect(result.kind).toBe('no-winner') + const events = await journal.loadTree('finalizer-refusal') + const managerSettlement = events?.find((event) => event.kind === 'settled') + expect(managerSettlement).toMatchObject({ + kind: 'settled', + status: 'done', + verdict: { valid: false }, + }) + }) + it('conserves the budget across depth: Σ spend over every tree ≤ the root ceiling', async () => { const journal = new InMemorySpawnJournal() const blobs = new InMemoryResultBlobStore() diff --git a/tests/kernel/inbox.test.ts b/tests/kernel/inbox.test.ts index 9d19f2a0..75e11792 100644 --- a/tests/kernel/inbox.test.ts +++ b/tests/kernel/inbox.test.ts @@ -5,10 +5,10 @@ import { type AgentSpec, createBudgetPool, createExecutor, createInbox } from '. describe('worker inbox (down-leg receive end)', () => { it('parses the down-message shapes; ignores malformed', () => { const inbox = createInbox() - inbox.deliver({ steer: 'do X' }) - inbox.deliver({ answer: 'use v2', questionId: 'q1' }) - inbox.deliver({ junk: true }) // ignored, never throws - inbox.deliver(null) + expect(inbox.deliver({ steer: 'do X' })).toBe(true) + expect(inbox.deliver({ answer: 'use v2', questionId: 'q1' })).toBe(true) + expect(inbox.deliver({ junk: true })).toBe(false) + expect(inbox.deliver(null)).toBe(false) const drained = inbox.drain() expect(drained).toEqual([ { kind: 'steer', text: 'do X', interrupt: false }, diff --git a/tests/kernel/nested-coordination-durability.test.ts b/tests/kernel/nested-coordination-durability.test.ts index 75297b3f..0b4bd67e 100644 --- a/tests/kernel/nested-coordination-durability.test.ts +++ b/tests/kernel/nested-coordination-durability.test.ts @@ -5,7 +5,11 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { fullProfileMaterialization } from '../../src/agent/profile-materialization' import type { CoordinationEvent, QuestionRecord } from '../../src/mcp/tools/coordination' import { supervise } from '../../src/runtime/supervise/supervise' -import type { DriveHarness } from '../../src/runtime/supervise/supervisor-agent' +import type { + DriveHarness, + DriveHarnessOwnerContext, +} from '../../src/runtime/supervise/supervisor-agent' +import type { ToolLoopChat } from '../../src/runtime/tool-loop' import { scriptedBrain } from './scripted-brain' async function callTool( @@ -136,4 +140,189 @@ describe('nested supervisor coordination durability', () => { } expect([...counts.values()].sort()).toEqual([2, 2]) }) + + it('routes concurrent manager steers through distinct owner-scoped harness sessions', async () => { + const contexts: DriveHarnessOwnerContext[] = [] + const delivered = new Map() + let startedCount = 0 + let bothStarted!: () => void + const bothManagersStarted = new Promise((resolve) => { + bothStarted = resolve + }) + const resolveDriveHarness = (context: DriveHarnessOwnerContext): DriveHarness => { + contexts.push(context) + let release!: () => void + const steered = new Promise((resolve) => { + release = resolve + }) + const harness: DriveHarness = async () => { + startedCount += 1 + if (startedCount === 2) bothStarted() + await steered + } + harness.deliver = (message): boolean => { + delivered.set(context.assignmentId ?? 'root', message) + release() + return true + } + return harness + } + let turn = 0 + const brain: ToolLoopChat = async () => { + turn += 1 + if (turn === 1) { + const manager = { + name: 'identical-manager', + harness: 'codex', + metadata: { role: 'driver' }, + } + return { + toolCalls: [ + { + id: 'spawn-a', + name: 'spawn_agent', + arguments: JSON.stringify({ profile: manager, task: 'same task', key: 'manager-a' }), + }, + { + id: 'spawn-b', + name: 'spawn_agent', + arguments: JSON.stringify({ profile: manager, task: 'same task', key: 'manager-b' }), + }, + ], + } + } + if (turn === 2) { + await bothManagersStarted + return { + toolCalls: [ + { + id: 'steer-a', + name: 'steer_agent', + arguments: JSON.stringify({ + workerId: 'owner-routed:s0', + instruction: 'instruction for manager A', + }), + }, + { + id: 'steer-b', + name: 'steer_agent', + arguments: JSON.stringify({ + workerId: 'owner-routed:s1', + instruction: 'instruction for manager B', + }), + }, + ], + } + } + if (turn <= 4) { + return { + toolCalls: [{ id: `await-${turn}`, name: 'await_event', arguments: JSON.stringify({}) }], + } + } + return { content: 'finished', toolCalls: [] } + } + + await supervise( + { name: 'root', harness: 'cli-base', prompt: { systemPrompt: 'Run both managers.' } }, + 'root task', + { + backend: { + backend: 'router', + routerBaseUrl: 'http://unused.invalid', + routerKey: 'unused', + model: 'unused/model', + }, + budget: { maxIterations: 16, maxTokens: 10_000 }, + perWorker: { maxIterations: 4, maxTokens: 1_000 }, + runId: 'owner-routed', + resolveDriveHarness, + driveHarnessMaterialization: fullProfileMaterialization, + brain, + maxTurns: 8, + }, + ) + + expect(contexts).toHaveLength(2) + expect(new Set(contexts.map((context) => context.ownerId)).size).toBe(2) + expect(contexts.map((context) => context.assignmentId).sort()).toEqual([ + 'key:manager-a', + 'key:manager-b', + ]) + for (const context of contexts) { + expect(Object.isFrozen(context)).toBe(true) + expect(Object.isFrozen(context.profile)).toBe(true) + expect(Object.isFrozen(context.identity)).toBe(true) + } + expect(delivered).toEqual( + new Map([ + ['key:manager-a', { steer: 'instruction for manager A', interrupt: false }], + ['key:manager-b', { steer: 'instruction for manager B', interrupt: false }], + ]), + ) + }) + + it('refuses one steerable harness instance reused across manager owners', async () => { + const sharedHarness: DriveHarness = async () => {} + sharedHarness.deliver = (): boolean => true + const seen: Array>> = [] + + await supervise( + { name: 'root', harness: 'cli-base', prompt: { systemPrompt: 'Run both managers.' } }, + 'root task', + { + backend: { + backend: 'router', + routerBaseUrl: 'http://unused.invalid', + routerKey: 'unused', + model: 'unused/model', + }, + budget: { maxIterations: 16, maxTokens: 10_000 }, + perWorker: { maxIterations: 4, maxTokens: 1_000 }, + runId: 'owner-reuse-refused', + resolveDriveHarness: () => sharedHarness, + driveHarnessMaterialization: fullProfileMaterialization, + brain: scriptedBrain( + [ + { + toolCalls: [ + { + name: 'spawn_agent', + arguments: { + profile: { + name: 'identical-manager', + harness: 'codex', + metadata: { role: 'driver' }, + }, + task: 'same task', + key: 'manager-a', + }, + }, + { + name: 'spawn_agent', + arguments: { + profile: { + name: 'identical-manager', + harness: 'codex', + metadata: { role: 'driver' }, + }, + task: 'same task', + key: 'manager-b', + }, + }, + ], + }, + { content: 'stop after reading the spawn results' }, + ], + seen, + ), + }, + ) + + expect(JSON.stringify(seen[1])).toContain( + 'steerable driveHarness is already bound to manager owner', + ) + expect(JSON.stringify(seen[1])).toContain( + 'resolveDriveHarness must return a distinct steerable instance', + ) + }) }) diff --git a/tests/kernel/spawn-forest.test.ts b/tests/kernel/spawn-forest.test.ts new file mode 100644 index 00000000..4e209023 --- /dev/null +++ b/tests/kernel/spawn-forest.test.ts @@ -0,0 +1,251 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { + contentAddress, + FileResultBlobStore, + FileSpawnJournal, +} from '../../src/durable/spawn-journal' +import { loadSpawnForest } from '../../src/runtime' +import { driverChild, withDriverExecutor } from '../../src/runtime/supervise/driver-executor' +import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { createSupervisor } from '../../src/runtime/supervise/supervisor' +import { nestedDriverTreeRoot } from '../../src/runtime/supervise/tree-key' +import type { + Agent, + AgentSpec, + Executor, + ExecutorResult, + Scope, + SpawnJournal, + UsageEvent, +} from '../../src/runtime/supervise/types' + +const rootBudget = { maxIterations: 10, maxTokens: 10_000 } +const childBudget = { maxIterations: 4, maxTokens: 1_000 } + +function leaf( + name: string, + out: unknown, + runtime: Executor['runtime'] = 'forest-leaf', +): Agent { + const executor: Executor = { + runtime, + execute(): AsyncIterable { + return (async function* () { + yield { kind: 'tokens', input: 3, output: 2 } + yield { kind: 'iteration' } + })() + }, + teardown: () => Promise.resolve({ destroyed: true }), + resultArtifact(): ExecutorResult { + return { + outRef: contentAddress(out), + out, + verdict: { valid: true, score: 1 }, + spent: { iterations: 1, tokens: { input: 3, output: 2 }, usd: 0, ms: 0 }, + } + }, + } + const spec: AgentSpec = { profile: { name } as AgentProfile, harness: null, executor } + return { name, act: async () => out, executorSpec: spec } as Agent & { + executorSpec: AgentSpec + } +} + +async function onlyDone(scope: Scope): Promise { + const settled = await scope.next() + if (settled?.kind !== 'done') throw new Error('expected one completed child') + return settled.out +} + +describe('loadSpawnForest', () => { + it('cold-loads every node and event from a file-backed root → driver → leaf run', async () => { + const dir = await mkdtemp(join(tmpdir(), 'spawn-forest-complete-')) + try { + const journalPath = join(dir, 'spawn-journal.jsonl') + const journal = new FileSpawnJournal(journalPath) + const blobs = new FileResultBlobStore(join(dir, 'blobs')) + const nested: Agent = { + name: 'nested', + async act(task, scope) { + const spawned = scope.spawn(leaf('leaf', { answer: 42 }), task, { + budget: childBudget, + label: 'leaf', + }) + if (!spawned.ok) throw new Error(spawned.reason) + return onlyDone(scope) + }, + } + const root: Agent = { + name: 'root', + async act(task, scope) { + const spawned = scope.spawn(driverChild('nested', nested, journal), task, { + budget: childBudget, + label: 'nested', + }) + if (!spawned.ok) throw new Error(spawned.reason) + return onlyDone(scope) + }, + } + + const result = await createSupervisor().run(root, 'solve', { + budget: rootBudget, + runId: 'forest-complete', + journal, + blobs, + executors: withDriverExecutor(createExecutorRegistry()), + maxDepth: 4, + }) + expect(result.kind).toBe('winner') + + const forest = await loadSpawnForest(new FileSpawnJournal(journalPath), 'forest-complete') + expect(forest.trees).toHaveLength(2) + expect(forest.nodes.map((node) => node.label).sort()).toEqual(['leaf', 'nested', 'root']) + expect(forest.nodes.find((node) => node.label === 'nested')?.ownedTreeRoot).toBe( + nestedDriverTreeRoot('forest-complete', 'forest-complete:s0'), + ) + expect(forest.events.length).toBeGreaterThan(forest.nodes.length) + expect(new Set(forest.events.map((entry) => entry.treeRoot))).toEqual( + new Set(forest.trees.map((tree) => tree.root)), + ) + expect(forest.inDoubt).toEqual([]) + expect(forest.missingTrees).toEqual([]) + expect(Object.isFrozen(forest)).toBe(true) + expect(Object.isFrozen(forest.nodes[0])).toBe(true) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + it('reports pending workers and an unopened driver subtree from a cold lost-work snapshot', async () => { + const dir = await mkdtemp(join(tmpdir(), 'spawn-forest-lost-')) + try { + const journalPath = join(dir, 'spawn-journal.jsonl') + const journal: SpawnJournal = new FileSpawnJournal(journalPath) + const at = new Date(0).toISOString() + await journal.beginTree('forest-lost', at) + await journal.appendEvent('forest-lost', { + kind: 'spawned', + id: 'forest-lost', + label: 'root', + budget: rootBudget, + runtime: 'inline', + seq: 0, + at, + }) + await journal.appendEvent('forest-lost', { + kind: 'spawned', + id: 'forest-lost:s0', + parent: 'forest-lost', + label: 'started-driver', + budget: childBudget, + runtime: 'driver', + ownedTreeRoot: nestedDriverTreeRoot('forest-lost', 'forest-lost:s0'), + seq: 0, + at, + }) + await journal.appendEvent('forest-lost', { + kind: 'spawned', + id: 'forest-lost:s1', + parent: 'forest-lost', + label: 'unopened-driver', + budget: childBudget, + runtime: 'driver', + ownedTreeRoot: nestedDriverTreeRoot('forest-lost', 'forest-lost:s1'), + seq: 1, + at, + }) + const nestedRoot = nestedDriverTreeRoot('forest-lost', 'forest-lost:s0') + await journal.beginTree(nestedRoot, at) + await journal.appendEvent(nestedRoot, { + kind: 'spawned', + id: 'forest-lost:s0:s0', + parent: 'forest-lost:s0', + label: 'lost-leaf', + budget: childBudget, + runtime: 'forest-leaf', + seq: 0, + at, + }) + + const forest = await loadSpawnForest(new FileSpawnJournal(journalPath), 'forest-lost') + expect(forest.trees).toHaveLength(2) + expect(forest.nodes.map((node) => [node.label, node.status])).toEqual( + expect.arrayContaining([ + ['started-driver', 'pending'], + ['unopened-driver', 'pending'], + ['lost-leaf', 'pending'], + ]), + ) + expect(forest.inDoubt.map((node) => node.label).sort()).toEqual([ + 'lost-leaf', + 'started-driver', + 'unopened-driver', + ]) + expect(forest.missingTrees).toEqual([ + expect.objectContaining({ ownerNodeId: 'forest-lost:s1' }), + ]) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + it('does not infer tree ownership from a caller leaf whose open runtime string is driver', async () => { + const dir = await mkdtemp(join(tmpdir(), 'spawn-forest-byo-driver-')) + try { + const journalPath = join(dir, 'spawn-journal.jsonl') + const journal = new FileSpawnJournal(journalPath) + const blobs = new FileResultBlobStore(join(dir, 'blobs')) + const root: Agent = { + name: 'root', + async act(task, scope) { + const spawned = scope.spawn( + leaf('caller-leaf-named-driver', { answer: 7 }, 'driver'), + task, + { + budget: childBudget, + label: 'caller-leaf-named-driver', + }, + ) + if (!spawned.ok) throw new Error(spawned.reason) + return onlyDone(scope) + }, + } + + const result = await createSupervisor().run(root, 'solve', { + budget: rootBudget, + runId: 'forest-byo-driver', + journal, + blobs, + executors: createExecutorRegistry(), + }) + expect(result.kind).toBe('winner') + + // A pre-field journal may also contain a convention-shaped tree. Absence of the trusted + // ownership field is authoritative: the cold reader must neither scan nor adopt it. + const at = new Date(0).toISOString() + const decoyRoot = nestedDriverTreeRoot('forest-byo-driver', 'forest-byo-driver:s0') + await journal.beginTree(decoyRoot, at) + await journal.appendEvent(decoyRoot, { + kind: 'spawned', + id: 'decoy:s0', + parent: 'forest-byo-driver:s0', + label: 'must-not-be-followed', + budget: childBudget, + runtime: 'forest-leaf', + seq: 0, + at, + }) + + const forest = await loadSpawnForest(new FileSpawnJournal(journalPath), 'forest-byo-driver') + expect(forest.trees.map((tree) => tree.root)).toEqual(['forest-byo-driver']) + expect(forest.nodes.map((node) => node.label)).toEqual(['root', 'caller-leaf-named-driver']) + expect(forest.missingTrees).toEqual([]) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/kernel/supervise-convenience.test.ts b/tests/kernel/supervise-convenience.test.ts index 333dfc93..1faaa4a0 100644 --- a/tests/kernel/supervise-convenience.test.ts +++ b/tests/kernel/supervise-convenience.test.ts @@ -10,6 +10,7 @@ import { supervise, workerFromBackend, } from '../../src/runtime/supervise/supervise' +import { createRootHandle } from '../../src/runtime/supervise/supervisor' import type { Agent, AgentSpec, @@ -148,6 +149,54 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ expect(teardownCalled).toBe(true) }) + it('attaches a caller RootHandle and folds a live steer before the router manager next thinks', async () => { + const handle = createRootHandle() + let entered!: () => void + const firstTurnEntered = new Promise((resolve) => { + entered = resolve + }) + let release!: () => void + const releaseFirstTurn = new Promise((resolve) => { + release = resolve + }) + const seen: Array>> = [] + let turn = 0 + const running = supervise({ name: 'root', harness: 'cli-base' }, 'solve it', { + budget, + rootHandle: handle, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + brain: async (messages) => { + seen.push(messages) + turn += 1 + if (turn === 1) { + entered() + await releaseFirstTurn + return { + toolCalls: [{ id: 'list', name: 'list_questions', arguments: JSON.stringify({}) }], + } + } + return { content: 'stopped after reading the steer', toolCalls: [] } + }, + }) + + await firstTurnEntered + expect(handle.deliver({ junk: true })).toBe(false) + expect(handle.deliver({ steer: 'also test the negative case', interrupt: false })).toBe(true) + release() + await running + + expect(seen).toHaveLength(2) + expect(seen[1]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + role: 'user', + content: expect.stringContaining('also test the negative case'), + }), + ]), + ) + expect(() => handle.deliver({ steer: 'after completion' })).toThrow() + }) + it('runDir makes the run durable and resumable; unset stays in-memory', async () => { const dir = await mkdtemp(join(tmpdir(), 'supervise-rundir-')) try { @@ -676,7 +725,7 @@ describe('supervise — the one-call convenience (defaults blobs/perWorker/journ model: 'safe', }, }), - ).toThrow(/requires a local bridge driverBackend or an explicit driveHarness/) + ).toThrow(/requires a local bridge driverBackend.*explicit driveHarness.*resolveDriveHarness/) }) it('allowedModels rejects a router model outside the allowed set', () => { diff --git a/tests/kernel/supervise-full-profile-bridge.test.ts b/tests/kernel/supervise-full-profile-bridge.test.ts index ce549294..4071ff58 100644 --- a/tests/kernel/supervise-full-profile-bridge.test.ts +++ b/tests/kernel/supervise-full-profile-bridge.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it } from 'vitest' import { InMemorySpawnJournal } from '../../src/durable/spawn-journal' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' import { supervise } from '../../src/runtime/supervise/supervise' +import { createRootHandle } from '../../src/runtime/supervise/supervisor' type BridgeRequest = { model: string @@ -115,9 +116,323 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( server = undefined }) + it('forcefully steers a live bridge root and resumes the same manager session', async () => { + const requests: BridgeRequest[] = [] + const cancelled: string[] = [] + let markFirstRequest!: () => void + const firstRequest = new Promise((resolve) => { + markFirstRequest = resolve + }) + server = createServer(async (req, res) => { + const cancelledId = cancelledRunId(req.url) + if (cancelledId !== undefined) { + cancelled.push(cancelledId) + respondWithTerminalCancellation(res, cancelledId) + return + } + const body = await readJson(req) + requests.push(body) + if (requests.length === 1) { + res.writeHead(200, { + 'content-type': 'text/event-stream', + 'x-run-id': body.run_id, + 'x-run-request-digest': TEST_RUN_DIGEST, + }) + res.write( + `id: 1\ndata: ${JSON.stringify({ usage: { prompt_tokens: 5, completion_tokens: 2, cost: 0.01 } })}\n\n`, + ) + markFirstRequest() + return + } + respondWithBridgeStream(res, body, successStream('corrected manager turn')) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const handle = createRootHandle() + const running = supervise( + { + name: 'pi-leader', + harness: 'codex', + prompt: { systemPrompt: 'Lead the pursuit.' }, + model: { default: 'gpt-5.6' }, + }, + 'Choose the next experiment.', + { + rootHandle: handle, + backend: { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + model: 'codex/gpt-5.6', + }, + budget: { maxIterations: 4, maxTokens: 10_000 }, + }, + ) + + await firstRequest + expect( + handle.deliver({ + steer: 'replace the weak plan with the falsification test', + interrupt: true, + }), + ).toBe(true) + await running + + expect(requests).toHaveLength(2) + expect(cancelled).toEqual([requests[0]?.run_id]) + expect(requests[1]?.session_id).toBe(requests[0]?.session_id) + expect(requests[1]?.messages).toEqual([ + { + role: 'user', + content: expect.stringContaining('replace the weak plan with the falsification test'), + }, + ]) + expect(() => handle.deliver({ steer: 'after completion' })).toThrow() + }) + + it('selects heterogeneous leaf completion checks from the exact authorized spawn context', async () => { + const requests: BridgeRequest[] = [] + server = createServer(async (req, res) => { + const body = await readJson(req) + requests.push(body) + const content = + body.agent_profile.name === 'worker' + ? 'WORK=42' + : body.agent_profile.name === 'evaluator' + ? 'EVAL=pass' + : 'FALLBACK=ready' + respondWithBridgeStream(res, body, successStream(content)) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const contexts: Array<{ + name: string | undefined + task: unknown + key: string | undefined + assignmentId: string + depth: number + frozen: boolean + candidateDigest: string | undefined + }> = [] + let turn = 0 + const result = await supervise( + { name: 'root', harness: 'cli-base', prompt: { systemPrompt: 'Run all checks.' } }, + 'Compare implementation and evaluation evidence.', + { + backend: { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + model: 'codex/test', + }, + budget: { maxIterations: 20, maxTokens: 20_000 }, + perWorker: { maxIterations: 4, maxTokens: 2_000 }, + deliverable: { + check: (out) => + typeof out === 'object' && + out !== null && + (out as { content?: unknown }).content === 'FALLBACK=ready', + describe: 'fallback artifact is ready', + }, + brain: async () => { + turn += 1 + if (turn === 1) { + return { + toolCalls: [ + { + id: 'worker', + name: 'spawn_agent', + arguments: JSON.stringify({ + profile: { name: 'worker' }, + task: { kind: 'implement', expected: 'WORK=42' }, + key: 'worker', + }), + }, + { + id: 'evaluator', + name: 'spawn_agent', + arguments: JSON.stringify({ + profile: { name: 'evaluator' }, + task: { kind: 'evaluate', expected: 'EVAL=pass' }, + key: 'evaluator', + }), + }, + { + id: 'fallback', + name: 'spawn_agent', + arguments: JSON.stringify({ + profile: { name: 'fallback' }, + task: { kind: 'archive', expected: 'FALLBACK=ready' }, + key: 'fallback', + }), + }, + ], + } + } + if (turn <= 4) { + return { + toolCalls: [ + { id: `await-${turn}`, name: 'await_event', arguments: JSON.stringify({}) }, + ], + } + } + return { content: 'done', toolCalls: [] } + }, + authorizeSpawn: (input) => ({ + profile: input.profile, + execution: { + candidateDigest: canonicalCandidateDigest({ candidate: input.profile.name }), + correlation: { campaign: 'heterogeneous-checks' }, + }, + }), + resolveDeliverable: (input) => { + contexts.push({ + name: input.profile.name, + task: input.task, + key: input.key, + assignmentId: input.assignmentId, + depth: input.depth, + frozen: + Object.isFrozen(input) && + Object.isFrozen(input.profile) && + Object.isFrozen(input.parent) && + Object.isFrozen(input.parentIdentity) && + Object.isFrozen(input.execution) && + Object.isFrozen(input.task) && + Object.isFrozen(input.budget), + candidateDigest: input.execution.candidateDigest, + }) + if (input.profile.name === 'fallback') return undefined + const expected = input.profile.name === 'worker' ? 'WORK=42' : 'EVAL=pass' + return { + check: (out) => + typeof out === 'object' && + out !== null && + (out as { content?: unknown }).content === expected, + describe: `${input.profile.name} emits ${expected}`, + } + }, + }, + ) + + expect(result.kind).toBe('winner') + expect(requests.map((request) => request.agent_profile.name).sort()).toEqual([ + 'evaluator', + 'fallback', + 'worker', + ]) + expect(contexts).toHaveLength(3) + expect(contexts.map((context) => context.name).sort()).toEqual([ + 'evaluator', + 'fallback', + 'worker', + ]) + expect(contexts.every((context) => context.frozen)).toBe(true) + expect(contexts.map((context) => context.depth)).toEqual([1, 1, 1]) + expect(contexts.map((context) => context.assignmentId).sort()).toEqual([ + 'key:evaluator', + 'key:fallback', + 'key:worker', + ]) + for (const context of contexts) { + expect(context.candidateDigest).toBe(canonicalCandidateDigest({ candidate: context.name })) + expect(context.task).toMatchObject({ expected: expect.any(String) }) + expect(context.key).toBe(context.name) + } + }) + + it('reuses the recorded per-spawn completion result on durable resume', async () => { + let requests = 0 + server = createServer(async (req, res) => { + const body = await readJson(req) + requests += 1 + respondWithBridgeStream(res, body, successStream('RESULT=durable')) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const runDir = await mkdtemp(join(tmpdir(), 'per-spawn-deliverable-resume-')) + let resolutions = 0 + const makeBrain = () => { + let turn = 0 + return async () => { + turn += 1 + if (turn === 1) { + return { + toolCalls: [ + { + id: 'spawn', + name: 'spawn_agent', + arguments: JSON.stringify({ + profile: { name: 'durable-worker' }, + task: 'produce the durable result', + key: 'durable-worker', + }), + }, + ], + } + } + if (turn === 2) { + return { + toolCalls: [{ id: 'await', name: 'await_event', arguments: JSON.stringify({}) }], + } + } + return { content: 'done', toolCalls: [] } + } + } + const common = { + backend: { + backend: 'bridge' as const, + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + model: 'codex/test', + }, + budget: { maxIterations: 8, maxTokens: 10_000 }, + perWorker: { maxIterations: 2, maxTokens: 1_000 }, + runDir, + runId: 'per-spawn-deliverable-resume', + resolveDeliverable: () => { + resolutions += 1 + return { + check: (out: unknown) => + typeof out === 'object' && + out !== null && + (out as { content?: unknown }).content === 'RESULT=durable', + } + }, + } + const profile = { name: 'root', harness: 'cli-base' as const } + try { + const first = await supervise(profile, 'resume the exact result', { + ...common, + brain: makeBrain(), + }) + const second = await supervise(profile, 'resume the exact result', { + ...common, + brain: makeBrain(), + }) + + expect(first.kind).toBe('winner') + expect(second.kind).toBe('winner') + expect(requests).toBe(1) + // The resumed manager re-authorizes and re-resolves the exact keyed request, but Scope returns + // the recorded settlement before constructing or executing a replacement backend worker. + expect(resolutions).toBe(2) + } finally { + await rm(runDir, { recursive: true, force: true }) + } + }) + it('runs PI → nested supervisor → worker without dropping any authored profile axis', async () => { const requests: BridgeRequest[] = [] const journal = new InMemorySpawnJournal() + const resolvedDeliverables: string[] = [] + const classifications: Array<{ + name: string | undefined + metadataRole: unknown + experimentId: string | undefined + frozen: boolean + isDriver: boolean + }> = [] const authorizations: Array<{ depth: number frozen: boolean @@ -156,7 +471,9 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( critic: { description: 'Find a confound', prompt: 'Challenge the result.' }, }, modes: { adversarial: { prompt: 'Try to falsify the claim.' } }, - metadata: { role: 'driver', depth: 1, family: 'scientific-method' }, + // Deliberately false model-authored authority: product authorization below makes + // this profile recursive even though the authored metadata calls it a worker. + metadata: { role: 'worker', depth: 1, family: 'scientific-method' }, }, task: 'Run one experiment and return its measured result.', }) @@ -183,7 +500,9 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( ], failOnError: true, }, - metadata: { role: 'worker', depth: 2, family: 'scientific-method' }, + // Deliberately false in the other direction: model-authored metadata cannot make a + // profile recursive when product authorization classifies it as a leaf. + metadata: { role: 'driver', depth: 2, family: 'scientific-method' }, }, task: 'Measure the system and report RESULT=42.', }) @@ -273,6 +592,29 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( }, } }, + isDriverProfile: (input) => { + const isDriver = input.execution.correlation?.experimentId === 'experiment-1' + classifications.push({ + name: input.profile.name, + metadataRole: input.profile.metadata?.role, + experimentId: input.execution.correlation?.experimentId, + frozen: + Object.isFrozen(input) && + Object.isFrozen(input.profile) && + Object.isFrozen(input.parent) && + Object.isFrozen(input.parentIdentity) && + Object.isFrozen(input.execution) && + Object.isFrozen(input.execution.correlation) && + Object.isFrozen(input.task) && + Object.isFrozen(input.budget), + isDriver, + }) + return isDriver + }, + resolveDeliverable: (input) => { + resolvedDeliverables.push(input.profile.name ?? 'unnamed') + return undefined + }, }) expect(result.kind).toBe('winner') @@ -328,6 +670,23 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( task: 'Measure the system and report RESULT=42.', }, ]) + expect(classifications).toEqual([ + { + name: 'methods-supervisor', + metadataRole: 'worker', + experimentId: 'experiment-1', + frozen: true, + isDriver: true, + }, + { + name: 'experiment-worker', + metadataRole: 'driver', + experimentId: 'experiment-2', + frozen: true, + isDriver: false, + }, + ]) + expect(resolvedDeliverables).toEqual(['experiment-worker']) const rootEvents = await journal.loadTree('identity-run') expect(JSON.stringify(rootEvents)).not.toContain(backend.bridgeUrl) diff --git a/tests/kernel/supervise.test.ts b/tests/kernel/supervise.test.ts index 69c8ba7c..71271bfb 100644 --- a/tests/kernel/supervise.test.ts +++ b/tests/kernel/supervise.test.ts @@ -9,6 +9,7 @@ import { import { ValidationError } from '../../src/errors' import { defaultSelectWinner } from '../../src/runtime/run-loop' import { createBudgetPool, spendFromUsageEvents } from '../../src/runtime/supervise/budget' +import { createInbox } from '../../src/runtime/supervise/inbox' import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' import { createScope, settledToIteration } from '../../src/runtime/supervise/scope' import { createRootHandle, createSupervisor } from '../../src/runtime/supervise/supervisor' @@ -70,7 +71,13 @@ function mockExecutor(script: MockScript): Executor { for (const ev of script.events) yield ev })() }, - ...(script.inbox ? { deliver: (m: unknown) => script.inbox?.push(m) } : {}), + ...(script.inbox + ? { + deliver(message: unknown): void { + script.inbox?.push(message) + }, + } + : {}), teardown(): Promise<{ destroyed: boolean }> { return Promise.resolve({ destroyed: true }) }, @@ -452,7 +459,7 @@ describe('reactive scope', () => { expect(() => pool.assertNoOpenTickets()).not.toThrow() }) - it('send() steers a LIVE child via its inbox; false for settled / unknown / no-inbox', async () => { + it('accepts a void-returning Executor.deliver and steers its live child', async () => { const { scope } = await beginScope() const inbox: unknown[] = [] let release!: () => void @@ -502,6 +509,39 @@ describe('reactive scope', () => { await scope.next() }) + it('send() returns false when a live child inbox rejects a malformed message', async () => { + const { scope } = await beginScope() + const inbox = createInbox() + let release!: () => void + const block = new Promise((resolve) => { + release = resolve + }) + const executor = mockExecutor({ out: 1, events: tokensOnly(1, 1, 1), block }) + executor.deliver = (message) => inbox.deliver(message) + const agent = { + name: 'validated-inbox', + act: async () => 1, + executorSpec: { + profile: { name: 'validated-inbox' } as AgentProfile, + harness: null, + executor, + }, + } as Agent & { executorSpec: AgentSpec } + const spawned = scope.spawn(agent, 'task', { + budget: { maxIterations: 1, maxTokens: 10 }, + label: 'validated-inbox', + }) + if (!spawned.ok) throw new Error('spawn should have succeeded') + + expect(scope.send(spawned.handle.id, { junk: true })).toBe(false) + expect(scope.send(spawned.handle.id, { steer: 'use the valid route' })).toBe(true) + expect(inbox.drain()).toEqual([ + { kind: 'steer', text: 'use the valid route', interrupt: false }, + ]) + release() + await scope.next() + }) + it('next() yields in monotonic seq order and view reflects the in-memory tree', async () => { const { scope } = await beginScope() for (let i = 0; i < 4; i += 1) { @@ -889,26 +929,205 @@ describe('supervisor', () => { const handle = createRootHandle() // Detached: every method is a typed throw, never a silent no-op. expect(() => handle.view()).toThrow() + expect(() => handle.deliver({ steer: 'too early' })).toThrow() const supervisor = createSupervisor() supervisor.attach(handle) let observed = -1 + let received: unknown + const started = deferred() + const release = deferred() const driver: Agent = { name: 'observe', + deliver(message): boolean { + received = message + return true + }, async act(_t, scope: Scope): Promise { scope.spawn(leafAgent('c', { out: 'c', events: tokensOnly(1, 1, 1) }), 't', { budget: { maxIterations: 1, maxTokens: 10 }, label: 'c', }) observed = handle.view().nodes.length + started.resolve() + await release.promise await scope.next() return 'c' }, } - const result = await supervisor.run(driver, 't', supervisorOpts()) + const running = supervisor.run(driver, 't', supervisorOpts()) + await started.promise + expect(handle.deliver({ steer: 'use the corrected plan', interrupt: true })).toBe(true) + release.resolve() + const result = await running expect(result.kind).toBe('winner') expect(observed).toBe(1) + expect(received).toEqual({ steer: 'use the corrected plan', interrupt: true }) // Unbound again after the run completes. expect(() => handle.view()).toThrow() + expect(() => handle.deliver({ steer: 'too late' })).toThrow() + }) + + it('a live root manager without an inbox reports that delivery was not accepted', async () => { + const handle = createRootHandle() + const supervisor = createSupervisor() + supervisor.attach(handle) + const started = deferred() + const release = deferred() + const running = supervisor.run( + { + name: 'no-inbox', + async act() { + started.resolve() + await release.promise + return 'done' + }, + }, + 't', + supervisorOpts(), + ) + await started.promise + expect(handle.deliver({ steer: 'cannot receive this' })).toBe(false) + release.resolve() + await running + }) + + it('RootHandle returns false when the live manager inbox rejects malformed input', async () => { + const handle = createRootHandle() + const supervisor = createSupervisor() + supervisor.attach(handle) + const inbox = createInbox() + const started = deferred() + const release = deferred() + const running = supervisor.run( + { + name: 'validated-root-inbox', + deliver: (message) => inbox.deliver(message), + async act() { + started.resolve() + await release.promise + return 'done' + }, + }, + 't', + supervisorOpts({ runId: 'validated-root-inbox' }), + ) + await started.promise + + expect(handle.deliver({ junk: true })).toBe(false) + expect(handle.deliver({ steer: 'valid correction' })).toBe(true) + expect(inbox.drain()).toEqual([{ kind: 'steer', text: 'valid correction', interrupt: false }]) + release.resolve() + await running + }) + + it('refuses to cross-route one RootHandle across concurrent runs', async () => { + const handle = createRootHandle() + const firstSupervisor = createSupervisor() + const secondSupervisor = createSupervisor() + firstSupervisor.attach(handle) + secondSupervisor.attach(handle) + + const firstStarted = deferred() + const releaseFirst = deferred() + const firstMessages: unknown[] = [] + const first = firstSupervisor.run( + { + name: 'first-owner', + deliver(message): boolean { + firstMessages.push(message) + return true + }, + async act() { + firstStarted.resolve() + await releaseFirst.promise + return 'first' + }, + }, + 't', + supervisorOpts({ runId: 'root-handle-first' }), + ) + await firstStarted.promise + + let secondActed = false + await expect( + secondSupervisor.run( + { + name: 'second-owner', + async act() { + secondActed = true + return 'second' + }, + }, + 't', + supervisorOpts({ runId: 'root-handle-second' }), + ), + ).rejects.toThrow(/already controls a live run/) + expect(secondActed).toBe(false) + expect(handle.deliver({ steer: 'still for first' })).toBe(true) + expect(firstMessages).toEqual([{ steer: 'still for first' }]) + + releaseFirst.resolve() + await expect(first).resolves.toMatchObject({ kind: 'winner', out: 'first' }) + + const secondStarted = deferred() + const releaseSecond = deferred() + const secondMessages: unknown[] = [] + const sequential = secondSupervisor.run( + { + name: 'second-owner-after-release', + deliver(message): boolean { + secondMessages.push(message) + return true + }, + async act() { + secondStarted.resolve() + await releaseSecond.promise + return 'second' + }, + }, + 't', + supervisorOpts({ runId: 'root-handle-sequential' }), + ) + await secondStarted.promise + expect(handle.deliver({ steer: 'now for second' })).toBe(true) + expect(secondMessages).toEqual([{ steer: 'now for second' }]) + releaseSecond.resolve() + await expect(sequential).resolves.toMatchObject({ kind: 'winner', out: 'second' }) + }) + + it('releases a RootHandle lease when startup fails before the manager runs', async () => { + const handle = createRootHandle() + const supervisor = createSupervisor() + supervisor.attach(handle) + const occupied = new InMemorySpawnJournal() + await occupied.beginTree('occupied-root-handle', new Date(0).toISOString()) + + await expect( + supervisor.run( + { name: 'never-starts', act: async () => 'unreachable' }, + 't', + supervisorOpts({ runId: 'occupied-root-handle', journal: occupied }), + ), + ).rejects.toThrow(/already exists/) + + const started = deferred() + const release = deferred() + const running = supervisor.run( + { + name: 'starts-after-failure', + async act() { + started.resolve() + await release.promise + return 'done' + }, + }, + 't', + supervisorOpts({ runId: 'root-handle-after-startup-failure' }), + ) + await started.promise + expect(handle.view().root).toBe('root-handle-after-startup-failure') + release.resolve() + await expect(running).resolves.toMatchObject({ kind: 'winner', out: 'done' }) }) it('attach rejects a foreign handle not minted by createRootHandle', () => { From 74b7560257acb7f77febbec1f5fcb36a7addee07 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 12:52:07 -0600 Subject: [PATCH 11/12] fix(supervise): export root control handle --- docs/api/index.md | 4 +- docs/api/primitive-catalog.md | 3 +- docs/api/runtime.md | 184 +++++++++++++++++++-- src/runtime/index.ts | 2 +- tests/kernel/supervise-convenience.test.ts | 2 +- 5 files changed, 175 insertions(+), 20 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index 1def30ad..f471a1a3 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -8025,8 +8025,8 @@ the terminal artifact is read from `resultArtifact()` after the stream drains. Optional inbox: receive an out-of-band message from the driver mid-run (the `send`/`steer_agent` verb). A streaming executor drains pending messages between turns and folds them into the next step (a steer / interrupt / resume). A one-shot executor that can't be steered mid-flight omits -this; `Scope.send` then returns `false` for it. Never throws — a malformed message is the -executor's to ignore. +this; `Scope.send` then returns `false` for it. Never throws — an inbox that rejects a malformed +message returns `false`, and that refusal propagates to the caller. ###### Parameters diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 9b9aefe4..b078e2d5 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -505,7 +505,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 643 exports. +Import from `@tangle-network/agent-runtime/kernel` — 644 exports. | Symbol | Kind | Summary | |---|---|---| @@ -548,6 +548,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 643 exports. | `createMcpEnvironment` | function | Wrap any MCP server as an `Environment`: `tools/list` becomes `AgenticTool[]` with provider-safe schemas; the domain supplies only the artifact lifecycle hooks. | | `createProgressTracker` | function | Build the settled-work ledger a `StopRule` decides from: record each settlement (idempotent by | | `createPushTraceSource` | function | A push source for OWNED tool loops (router-tools / cli-bridge tool dispatch): the loop calls | +| `createRootHandle` | function | Mint a `RootHandle` plus its supervisor-private control. The handle is the substrate a | | `createSandboxLineage` | function | Build a lineage bound to one client + its probed capabilities. The | | `createSandboxToolPartState` | function | Fresh per-turn {@link SandboxToolPartState} for {@link mapSandboxToolEvent} — an | | `createScope` | function | Create the reactive `Scope` a driver's `Agent.act` runs inside: spawn children on an atomically reserved conserved budget, settle via the `next()` cursor, journal for replay. | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index e6a71382..5ec4716c 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -777,6 +777,16 @@ One flattened node with the journal tree that owns its records. [`NodeSnapshot`](#nodesnapshot).[`budget`](#budget-17) +##### ownedTreeRoot? + +> `readonly` `optional` **ownedTreeRoot?**: `string` + +Exact nested journal tree owned by this node, when Runtime attested recursive ownership. + +###### Inherited from + +[`NodeSnapshot`](#nodesnapshot).[`ownedTreeRoot`](#ownedtreeroot-1) + ##### assignmentId? > `readonly` `optional` **assignmentId?**: `string` @@ -10617,7 +10627,8 @@ and the worker's agent loop drains them at two points (Drew's two delivery modes in-flight turn immediately, then re-plan with the message folded in — breaking the worker out of a wrong path mid-task instead of waiting for it to finish the step. -`deliver` never throws — a malformed message is ignored, per the `Executor.deliver` contract. +`deliver` never throws — a malformed message is ignored and returns `false`, so no caller can +report delivery for bytes this inbox discarded. #### Properties @@ -10657,9 +10668,10 @@ Present for an `answer` — the question id it resolves. ##### deliver() -> **deliver**(`msg`): `void` +> **deliver**(`msg`): `boolean` -The `Executor.deliver` implementation — accept a raw down-message from `Scope.send`. +The `Executor.deliver` implementation. Returns false when the raw message is malformed and +therefore was not queued; callers must not acknowledge a message this inbox discarded. ###### Parameters @@ -10669,7 +10681,7 @@ The `Executor.deliver` implementation — accept a raw down-message from `Scope. ###### Returns -`void` +`boolean` ##### drain() @@ -13896,6 +13908,12 @@ The rehydrated settlement; absent exactly when `state` is `'in-doubt'`. > `readonly` **budget**: [`Budget`](index.md#budget-4) +##### ownedTreeRoot? + +> `readonly` `optional` **ownedTreeRoot?**: `string` + +Exact nested journal tree owned by this node, when Runtime attested recursive ownership. + ##### assignmentId? > `readonly` `optional` **assignmentId?**: `string` @@ -14191,9 +14209,11 @@ Lifecycle stream sink, threaded into the root `Scope` so every `spawn`/settle em ### RootHandle -Live root handle — the substrate a chat/pi-viz client attaches to (Q2). `deliver` - sends a raw steer/answer to a manager inbox, `signal` controls the run, and `view()` - materializes the tree. +Live root handle — a chat/pi-viz client uses it to inspect and control one root run. + +#### Extended by + +- [`SteerableRootHandle`](#steerableroothandle) #### Type Parameters @@ -14220,12 +14240,12 @@ Phantom: binds the handle to the supervised run's output type. Type-only — nev [`TreeView`](#treeview) -##### deliver() +##### deliver()? -> **deliver**(`msg`): `boolean` +> `optional` **deliver**(`msg`): `boolean` -Deliver a raw down-message to the live root manager. Returns `false` when that manager has no -inbox. Before binding and after completion this fails loud like the other handle methods. +Optional for structural compatibility with existing view/signal/abort wrappers. Handles +minted by `createRootHandle` implement the required form in `SteerableRootHandle`. ###### Parameters @@ -14267,6 +14287,107 @@ inbox. Before binding and after completion this fails loud like the other handle *** +### SteerableRootHandle + +A Runtime-minted root handle that can deliver raw steering or answers to a live manager inbox. +Delivery returns `false` when the manager has no receive path; detached calls fail loud. + +#### Extends + +- [`RootHandle`](#roothandle-1)\<`Out`\> + +#### Type Parameters + +##### Out + +`Out` + +#### Properties + +##### \_\_out? + +> `readonly` `optional` **\_\_out?**: `Out` + +Phantom: binds the handle to the supervised run's output type. Type-only — never + present at runtime; lets `attach(h: RootHandle)` stay output-typed. + +###### Inherited from + +[`RootHandle`](#roothandle-1).[`__out`](#__out-1) + +#### Methods + +##### view() + +> **view**(): [`TreeView`](#treeview) + +###### Returns + +[`TreeView`](#treeview) + +###### Inherited from + +[`RootHandle`](#roothandle-1).[`view`](#view-3) + +##### signal() + +> **signal**(`msg`): `void` + +###### Parameters + +###### msg + +[`RootSignal`](#rootsignal) + +###### Returns + +`void` + +###### Inherited from + +[`RootHandle`](#roothandle-1).[`signal`](#signal-15) + +##### abort() + +> **abort**(`reason?`): `void` + +###### Parameters + +###### reason? + +`string` + +###### Returns + +`void` + +###### Inherited from + +[`RootHandle`](#roothandle-1).[`abort`](#abort-1) + +##### deliver() + +> **deliver**(`msg`): `boolean` + +Optional for structural compatibility with existing view/signal/abort wrappers. Handles +minted by `createRootHandle` implement the required form in `SteerableRootHandle`. + +###### Parameters + +###### msg + +`unknown` + +###### Returns + +`boolean` + +###### Overrides + +[`RootHandle`](#roothandle-1).[`deliver`](#deliver-3) + +*** + ### WidenGate The progressive-widening gate (MCTS-PW). Decides whether a settled child is @@ -17795,7 +17916,7 @@ adoption state; none of the built-ins can today. ### SpawnEvent -> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-4); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-4); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-4); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-4); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-4); `reason`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-4); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-4); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} +> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-4); `ownedTreeRoot?`: [`NodeId`](#nodeid-4); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-4); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-4); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-4); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-4); `reason`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-4); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-4); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO timestamp for human inspection only (NOT a replay input). @@ -17804,7 +17925,7 @@ Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO ##### Type Literal -\{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-4); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-4); `ownedTreeRoot?`: [`NodeId`](#nodeid-4); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} ###### kind @@ -17843,6 +17964,14 @@ Manager-scoped assignment identity used to join unkeyed and keyed work alike. > **runtime**: [`Runtime`](#runtime-4) +###### ownedTreeRoot? + +> `optional` **ownedTreeRoot?**: [`NodeId`](#nodeid-4) + +Exact nested journal tree this node owns. Runtime writes this only after privately +attesting the executor as a recursive scope owner. Its absence means no tree is followed, +including records written before this field existed and caller leaves named `driver`. + ###### identity? > `optional` **identity?**: [`NodeExecutionIdentity`](#nodeexecutionidentity) @@ -18609,8 +18738,11 @@ Stable content address shared by result and trace artifacts. Load every journal tree owned by one recursive supervision run and flatten its nodes/events. Nested driver tree keys are a Runtime implementation detail; callers should use this reader -instead of deriving or scanning keys themselves. The reader follows raw `spawned` records whose -runtime is `driver`, preserving each tree's independent cursor namespace on flattened events. +instead of deriving or scanning keys themselves. The reader follows only the explicit +`ownedTreeRoot` written after Runtime privately attested a recursive executor; the open runtime +string `driver` is never treated as ownership. Legacy records without `ownedTreeRoot` are +intentionally treated as leaves rather than guessing or scanning convention-derived keys. +This preserves each tree's independent cursor namespace on flattened events. A driver whose subtree was never begun is reported in `missingTrees`; any spawned non-root node without a terminal record is reported in `inDoubt`, matching resume's conservative lost-work interpretation. @@ -22638,6 +22770,28 @@ Create a supervisor that owns one recursive agent execution tree. *** +### createRootHandle() + +> **createRootHandle**\<`Out`\>(): [`SteerableRootHandle`](#steerableroothandle)\<`Out`\> + +Mint a `RootHandle` plus its supervisor-private control. The handle is the substrate a +chat/pi-viz client attaches to (Q2): `view()` reads the live tree, `signal()` delivers +an out-of-band message, `abort()` cascades. Before `run` binds it (and after `run` +unbinds it) the handle is fail-loud: a client that talks to a handle that is not +driving a live run gets a typed error, never a silent no-op. + +#### Type Parameters + +##### Out + +`Out` + +#### Returns + +[`SteerableRootHandle`](#steerableroothandle)\<`Out`\> + +*** + ### captureWorkerTraceEvidence() > **captureWorkerTraceEvidence**(`readSource`, `blobs`, `executed`): `Promise`\<[`WorkerTraceEvidence`](index.md#workertraceevidence)\> diff --git a/src/runtime/index.ts b/src/runtime/index.ts index c9cecd0d..29353f98 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -709,7 +709,7 @@ export { supervise, workerFromBackend, } from './supervise/supervise' -export { createSupervisor } from './supervise/supervisor' +export { createRootHandle, createSupervisor } from './supervise/supervisor' // Build a supervisor FROM its profile: the brain is resolved from `profile.harness` like // `createExecutor({backend})` resolves a worker — omitted/`cli-base` → the in-process router tool-loop, // a coding-CLI harness → a sandboxed harness driving the coordination verbs. No hand-built brain. diff --git a/tests/kernel/supervise-convenience.test.ts b/tests/kernel/supervise-convenience.test.ts index 1faaa4a0..583758d8 100644 --- a/tests/kernel/supervise-convenience.test.ts +++ b/tests/kernel/supervise-convenience.test.ts @@ -4,13 +4,13 @@ import { join } from 'node:path' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemorySpawnJournal } from '../../src/durable/spawn-journal' +import { createRootHandle } from '../../src/runtime/index' import type { ExecutorConfig } from '../../src/runtime/supervise/runtime' import { type SuperviseOptions, supervise, workerFromBackend, } from '../../src/runtime/supervise/supervise' -import { createRootHandle } from '../../src/runtime/supervise/supervisor' import type { Agent, AgentSpec, From 162fea5ae9aaa1dcb92b4967d29f5f558d344bcf Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 13:17:06 -0600 Subject: [PATCH 12/12] fix(supervise): cancel product tool invocations --- docs/api/primitive-catalog.md | 3 +- docs/api/runtime.md | 158 ++++++++++++++--- docs/canonical-api.md | 1 + src/runtime/index.ts | 1 + src/runtime/supervise/scope.ts | 4 +- src/runtime/supervise/supervise.ts | 4 +- src/runtime/supervise/supervisor-agent.ts | 28 ++- tests/kernel/supervisor-agent.test.ts | 199 +++++++++++++++++++++- 8 files changed, 362 insertions(+), 36 deletions(-) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index b078e2d5..625aff0d 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -505,7 +505,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 644 exports. +Import from `@tangle-network/agent-runtime/kernel` — 645 exports. | Symbol | Kind | Summary | |---|---|---| @@ -902,6 +902,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 644 exports. | `Supervisor` | interface | Owns the conserved pool, the spawn log, the abort cascade, the OTP intensity breaker, | | `SupervisorNodeContext` | interface | Trusted run/node identity Runtime binds to one manager. Model-authored tool arguments cannot | | `SupervisorToolDescriptor` | interface | One product-owned tool. It reuses the canonical MCP descriptor fields while Runtime supplies | +| `SupervisorToolInvocationContext` | interface | Trusted context for one product-tool invocation. The node identity remains the same detached, | | `SurfaceWorkerConfig` | interface | How a worker runs the surface task (its router substrate + per-attempt bounds). | | `SurfaceWorkerOut` | interface | What a surface worker settles with — the surface verdict the driver + deliverable read. `resolved` is | | `ToolLoopCompaction` | interface | Self-compaction — bound the loop's OWN context window the way a fresh-respawn (dumb-Ralph) loop | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 5ec4716c..2a3ac08e 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -795,7 +795,7 @@ Manager-scoped assignment identity, including deterministic ids for unkeyed sibl ###### Inherited from -[`NodeSnapshot`](#nodesnapshot).[`assignmentId`](#assignmentid-6) +[`NodeSnapshot`](#nodesnapshot).[`assignmentId`](#assignmentid-7) ##### identity? @@ -803,7 +803,7 @@ Manager-scoped assignment identity, including deterministic ids for unkeyed sibl ###### Inherited from -[`NodeSnapshot`](#nodesnapshot).[`identity`](#identity-5) +[`NodeSnapshot`](#nodesnapshot).[`identity`](#identity-6) ##### materialization? @@ -12484,6 +12484,8 @@ full-profile contract. Resolve product-owned tools from the exact trusted manager context. The same descriptors and handlers are bound to router and external-harness managers; resolution happens once per node. +Each handler receives that manager scope's live cancellation signal in its trusted invocation +context, including recursive parent and root cascades. ##### onCoordinationEvent? @@ -12789,58 +12791,162 @@ Exact trusted context after a manager-authored spawn has passed product authoriz Trusted run/node identity Runtime binds to one manager. Model-authored tool arguments cannot provide or replace any of these fields. +#### Extended by + +- [`SupervisorToolInvocationContext`](#supervisortoolinvocationcontext) + +#### Properties + +##### runId + +> `readonly` **runId**: `string` + +##### runNamespace + +> `readonly` **runNamespace**: `string` + +Stable across a durable restart; unique per in-memory invocation. + +##### nodeId + +> `readonly` **nodeId**: `string` + +Concrete Scope node that owns this manager's coordination stream. + +##### ownerId + +> `readonly` **ownerId**: `string` + +Stable identity of this manager's coordination stream. + +##### depth + +> `readonly` **depth**: `number` + +##### identity + +> `readonly` **identity**: [`NodeExecutionIdentity`](#nodeexecutionidentity) + +##### assignmentId? + +> `readonly` `optional` **assignmentId?**: `string` + +Assignment identity within the parent manager; absent only for the root. + +##### profile + +> `readonly` **profile**: `AgentProfile` + +##### task + +> `readonly` **task**: `unknown` + +*** + +### SupervisorToolInvocationContext + +Trusted context for one product-tool invocation. The node identity remains the same detached, +immutable snapshot supplied to the resolver; `signal` is the one live control reference Runtime +adds. It aborts when this manager's scope is cancelled by the caller, RootHandle, deadline, +breaker, or a recursive parent. + +#### Extends + +- [`SupervisorNodeContext`](#supervisornodecontext) + #### Properties ##### runId > `readonly` **runId**: `string` +###### Inherited from + +[`SupervisorNodeContext`](#supervisornodecontext).[`runId`](#runid-12) + ##### runNamespace > `readonly` **runNamespace**: `string` Stable across a durable restart; unique per in-memory invocation. +###### Inherited from + +[`SupervisorNodeContext`](#supervisornodecontext).[`runNamespace`](#runnamespace) + ##### nodeId > `readonly` **nodeId**: `string` Concrete Scope node that owns this manager's coordination stream. +###### Inherited from + +[`SupervisorNodeContext`](#supervisornodecontext).[`nodeId`](#nodeid-2) + ##### ownerId > `readonly` **ownerId**: `string` Stable identity of this manager's coordination stream. +###### Inherited from + +[`SupervisorNodeContext`](#supervisornodecontext).[`ownerId`](#ownerid-1) + ##### depth > `readonly` **depth**: `number` +###### Inherited from + +[`SupervisorNodeContext`](#supervisornodecontext).[`depth`](#depth-2) + ##### identity > `readonly` **identity**: [`NodeExecutionIdentity`](#nodeexecutionidentity) +###### Inherited from + +[`SupervisorNodeContext`](#supervisornodecontext).[`identity`](#identity-1) + ##### assignmentId? > `readonly` `optional` **assignmentId?**: `string` Assignment identity within the parent manager; absent only for the root. +###### Inherited from + +[`SupervisorNodeContext`](#supervisornodecontext).[`assignmentId`](#assignmentid-3) + ##### profile > `readonly` **profile**: `AgentProfile` +###### Inherited from + +[`SupervisorNodeContext`](#supervisornodecontext).[`profile`](#profile-5) + ##### task > `readonly` **task**: `unknown` +###### Inherited from + +[`SupervisorNodeContext`](#supervisornodecontext).[`task`](#task-25) + +##### signal + +> `readonly` **signal**: `AbortSignal` + *** ### SupervisorToolDescriptor One product-owned tool. It reuses the canonical MCP descriptor fields while Runtime supplies - the trusted node context as a separate argument and binds the result for either transport. + the trusted invocation context as a separate argument and binds the result for either + transport. Existing handlers remain compatible: the second argument only gains `signal`. #### Extends @@ -12884,7 +12990,7 @@ One product-owned tool. It reuses the canonical MCP descriptor fields while Runt ###### context -[`SupervisorNodeContext`](#supervisornodecontext) +[`SupervisorToolInvocationContext`](#supervisortoolinvocationcontext) ###### Returns @@ -14345,7 +14451,7 @@ Phantom: binds the handle to the supervised run's output type. Type-only — nev ###### Inherited from -[`RootHandle`](#roothandle-1).[`signal`](#signal-15) +[`RootHandle`](#roothandle-1).[`signal`](#signal-16) ##### abort() @@ -17263,7 +17369,7 @@ judge/verdict/score scheme is rejected. Fail loud — a tainted finding aborts. ##### root -[`NodeId`](#nodeid-4) +[`NodeId`](#nodeid-5) ##### options? @@ -17893,7 +17999,7 @@ Fail-closed spawn rejections: an exhausted pool, an exceeded recursion ceiling, ### SpawnPrior -> **SpawnPrior**\<`Out`\> = \{ `state`: `"completed"`; `settled`: [`Settled`](index.md#settled)\<`Out`\> & `object`; \} \| \{ `state`: `"retried"`; `priorId`: [`NodeId`](#nodeid-4); `reason`: `string`; \} \| \{ `state`: `"lost"`; `priorId`: [`NodeId`](#nodeid-4); \} +> **SpawnPrior**\<`Out`\> = \{ `state`: `"completed"`; `settled`: [`Settled`](index.md#settled)\<`Out`\> & `object`; \} \| \{ `state`: `"retried"`; `priorId`: [`NodeId`](#nodeid-5); `reason`: `string`; \} \| \{ `state`: `"lost"`; `priorId`: [`NodeId`](#nodeid-5); \} What a KEYED spawn resolved to when the key had a prior attempt. Absent on a fresh key (and on every unkeyed spawn). `'completed'` is the exactly-once path: NOTHING was spawned — the handle @@ -17916,7 +18022,7 @@ adoption state; none of the built-ins can today. ### SpawnEvent -> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-4); `ownedTreeRoot?`: [`NodeId`](#nodeid-4); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-4); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-4); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-4); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-4); `reason`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-4); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-4); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} +> **SpawnEvent** = \{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-5); `parent?`: [`NodeId`](#nodeid-5); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-4); `ownedTreeRoot?`: [`NodeId`](#nodeid-5); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-5); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-5); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-5); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-5); `reason`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-5); `parent?`: [`NodeId`](#nodeid-5); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-5); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} \| \{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-5); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO timestamp for human inspection only (NOT a replay input). @@ -17925,7 +18031,7 @@ Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO ##### Type Literal -\{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-4); `ownedTreeRoot?`: [`NodeId`](#nodeid-4); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"spawned"`; `id`: [`NodeId`](#nodeid-5); `parent?`: [`NodeId`](#nodeid-5); `label`: `string`; `key?`: `string`; `assignmentId?`: `string`; `budget`: [`Budget`](index.md#budget-4); `runtime`: [`Runtime`](#runtime-4); `ownedTreeRoot?`: [`NodeId`](#nodeid-5); `identity?`: [`NodeExecutionIdentity`](#nodeexecutionidentity); `seq`: `number`; `at`: `string`; \} ###### kind @@ -17933,11 +18039,11 @@ Journaled spawn-tree events (B1/B2). `seq` is the cursor order; `at` is an ISO ###### id -> **id**: [`NodeId`](#nodeid-4) +> **id**: [`NodeId`](#nodeid-5) ###### parent? -> `optional` **parent?**: [`NodeId`](#nodeid-4) +> `optional` **parent?**: [`NodeId`](#nodeid-5) ###### label @@ -17966,7 +18072,7 @@ Manager-scoped assignment identity used to join unkeyed and keyed work alike. ###### ownedTreeRoot? -> `optional` **ownedTreeRoot?**: [`NodeId`](#nodeid-4) +> `optional` **ownedTreeRoot?**: [`NodeId`](#nodeid-5) Exact nested journal tree this node owns. Runtime writes this only after privately attesting the executor as a recursive scope owner. Its absence means no tree is followed, @@ -17990,7 +18096,7 @@ Exact profile/task digests plus trusted candidate/campaign attribution when avai ##### Type Literal -\{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-4); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"execution-bound"`; `id`: [`NodeId`](#nodeid-5); `binding`: [`ExecutionBindingReceipt`](#executionbindingreceipt); `seq`: `number`; `at`: `string`; \} ###### kind @@ -18001,7 +18107,7 @@ only by digest; descriptor fields are safe structural labels, never credential-b ###### id -> **id**: [`NodeId`](#nodeid-4) +> **id**: [`NodeId`](#nodeid-5) ###### binding @@ -18019,7 +18125,7 @@ only by digest; descriptor fields are safe structural labels, never credential-b ##### Type Literal -\{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-4); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"materialized"`; `id`: [`NodeId`](#nodeid-5); `receipt`: [`ProfileMaterializationReceipt`](#profilematerializationreceipt); `seq`: `number`; `at`: `string`; \} ###### kind @@ -18029,7 +18135,7 @@ Trusted runtime transformation from the authorized profile to actual wire bytes. ###### id -> **id**: [`NodeId`](#nodeid-4) +> **id**: [`NodeId`](#nodeid-5) ###### receipt @@ -18047,7 +18153,7 @@ Trusted runtime transformation from the authorized profile to actual wire bytes. ##### Type Literal -\{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-4); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"settled"`; `id`: [`NodeId`](#nodeid-5); `status`: `"done"` \| `"down"`; `outRef?`: `string`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); `infra?`: `boolean`; `reason?`: `string`; `trace?`: [`WorkerTraceEvidence`](index.md#workertraceevidence); `seq`: `number`; `at`: `string`; \} ###### kind @@ -18055,7 +18161,7 @@ Trusted runtime transformation from the authorized profile to actual wire bytes. ###### id -> **id**: [`NodeId`](#nodeid-4) +> **id**: [`NodeId`](#nodeid-5) ###### status @@ -18104,13 +18210,13 @@ Structured tool evidence. Optional only for journals written before trace captur ##### Type Literal -\{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-4); `reason`: `string`; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"cancelled"`; `id`: [`NodeId`](#nodeid-5); `reason`: `string`; `seq`: `number`; `at`: `string`; \} *** ##### Type Literal -\{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-4); `parent?`: [`NodeId`](#nodeid-4); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"waiting"`; `id`: [`NodeId`](#nodeid-5); `parent?`: [`NodeId`](#nodeid-5); `label`: `string`; `spec`: [`WaitSpec`](#waitspec); `armedAt`: `number`; `seq`: `number`; `at`: `string`; \} ###### kind @@ -18123,11 +18229,11 @@ A wait-state node was ARMED. Lives in the SPAWN-ORDINAL namespace (`seq` is the ###### id -> **id**: [`NodeId`](#nodeid-4) +> **id**: [`NodeId`](#nodeid-5) ###### parent? -> `optional` **parent?**: [`NodeId`](#nodeid-4) +> `optional` **parent?**: [`NodeId`](#nodeid-5) ###### label @@ -18153,7 +18259,7 @@ A wait-state node was ARMED. Lives in the SPAWN-ORDINAL namespace (`seq` is the ##### Type Literal -\{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-4); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"woken"`; `id`: [`NodeId`](#nodeid-5); `by`: `"fired"` \| `"timeout"` \| `"cancelled"`; `outRef?`: `string`; `seq`: `number`; `at`: `string`; \} ###### kind @@ -18166,7 +18272,7 @@ A wait-state node SETTLED — the cursor-namespace twin of `settled`, kept disti ###### id -> **id**: [`NodeId`](#nodeid-4) +> **id**: [`NodeId`](#nodeid-5) ###### by @@ -18188,7 +18294,7 @@ A wait-state node SETTLED — the cursor-namespace twin of `settled`, kept disti ##### Type Literal -\{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-4); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} +\{ `kind`: `"metered"`; `id`: [`NodeId`](#nodeid-5); `spend`: [`Spend`](index.md#spend); `seq`: `number`; `at`: `string`; \} ###### kind @@ -18204,7 +18310,7 @@ A driver's OWN inference spend, journaled separately from spawned-child work — ###### id -> **id**: [`NodeId`](#nodeid-4) +> **id**: [`NodeId`](#nodeid-5) ###### spend diff --git a/docs/canonical-api.md b/docs/canonical-api.md index b728916b..0de27781 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -59,6 +59,7 @@ By default, a child with `metadata.role: 'driver'` becomes another supervisor ov Products that keep authority outside model-authored metadata pass `isDriverProfile`, which receives the exact frozen post-authorization profile, task, assignment, parent identity, and child execution identity before Runtime selects manager versus leaf. `resolveDeliverable` receives that same context only for backend-derived leaves, allowing each authorized assignment to carry its own completion check while `deliverable` remains the run-wide fallback. An attached `rootHandle` exposes the live root tree, control signals, cancellation, and raw steering through the root manager's real inbox; delivery returns `false` when that manager has no inbox and fails loudly outside the handle's live binding. +Product tools returned by `resolveSupervisorTools` receive a frozen `SupervisorToolInvocationContext`; its `signal` is the live manager-scope cancellation signal, so an abort-aware operation can stop on caller, root-handle, deadline, breaker, or recursive-parent cancellation through either the router or external-MCP path. The parent reserves that driver's allocation once; the nested manager partitions only that allocation, so nested work is neither double-charged nor allowed to borrow outside its branch. For an external-harness supervisor, the automatic path is local CLI bridge execution: Runtime adds the live coordination MCP under the reserved `agent-runtime-coordination` alias and executes the rest of the profile unchanged through a `bridge` `driverBackend ?? backend`. Before that runtime starts, the spawn journal records a `materialized` receipt containing the authored-profile, effective-profile, and platform-attachment digests. diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 29353f98..79794e62 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -724,6 +724,7 @@ export { type SupervisorNodeContextSeed, type SupervisorProfile, type SupervisorToolDescriptor, + type SupervisorToolInvocationContext, supervisorAgent, } from './supervise/supervisor-agent' export { diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index e98f9cfa..2f4db7f8 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -578,8 +578,8 @@ export function createScope(args: ScopeArgs): Scope { // the executor's signal into its acquireSandbox find-by-name reap, so an acquiring node // never leaks. const controller = new AbortController() - cascadeAbort = () => controller.abort() - if (args.signal.aborted) controller.abort() + cascadeAbort = () => controller.abort(args.signal.reason) + if (args.signal.aborted) controller.abort(args.signal.reason) else args.signal.addEventListener('abort', cascadeAbort, { once: true }) if (childDeadlineAtMs !== undefined) { clearChildDeadline = armDeadlineTimer(Math.max(0, childDeadlineAtMs - now()), () => diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index 66be1827..b8276a01 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -538,7 +538,9 @@ export interface SuperviseOptions { * full-profile contract. */ readonly driveHarnessMaterialization?: ProfileMaterializationContract /** Resolve product-owned tools from the exact trusted manager context. The same descriptors and - * handlers are bound to router and external-harness managers; resolution happens once per node. */ + * handlers are bound to router and external-harness managers; resolution happens once per node. + * Each handler receives that manager scope's live cancellation signal in its trusted invocation + * context, including recursive parent and root cascades. */ readonly resolveSupervisorTools?: ResolveSupervisorTools /** Awaited product transaction hook for every coordination record. `eventId` is stable across a * lost acknowledgement and durable restart; the record is not pull-visible until this commits. */ diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index 4bb80454..7b7df6b1 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -89,10 +89,19 @@ export interface SupervisorNodeContext { /** Context known before `Agent.act`; Runtime adds the concrete node, profile, and task. */ export type SupervisorNodeContextSeed = Omit +/** Trusted context for one product-tool invocation. The node identity remains the same detached, + * immutable snapshot supplied to the resolver; `signal` is the one live control reference Runtime + * adds. It aborts when this manager's scope is cancelled by the caller, RootHandle, deadline, + * breaker, or a recursive parent. */ +export interface SupervisorToolInvocationContext extends SupervisorNodeContext { + readonly signal: AbortSignal +} + /** One product-owned tool. It reuses the canonical MCP descriptor fields while Runtime supplies - * the trusted node context as a separate argument and binds the result for either transport. */ + * the trusted invocation context as a separate argument and binds the result for either + * transport. Existing handlers remain compatible: the second argument only gains `signal`. */ export interface SupervisorToolDescriptor extends Omit { - readonly handler: (raw: unknown, context: SupervisorNodeContext) => Promise + readonly handler: (raw: unknown, context: SupervisorToolInvocationContext) => Promise } /** Product policy for the tools one exact supervisor node may call. Resolved once per node. */ @@ -297,7 +306,9 @@ export function supervisorAgent( : undefined const priorCoordination = await deps.loadPriorCoordination?.() const nodeTools = - resolveTools && context ? await bindSupervisorTools(resolveTools, context) : undefined + resolveTools && context + ? await bindSupervisorTools(resolveTools, context, scope.signal) + : undefined const onEvent = bindSupervisorNodeObserver(context, observeNodeEvent, deps.onEvent) return build(priorCoordination, nodeTools, onEvent).act(task, scope) }, @@ -329,7 +340,9 @@ export function supervisorAgent( ? await deps.loadPriorCoordination() : deps.priorCoordination const nodeTools = - resolveTools && context ? await bindSupervisorTools(resolveTools, context) : undefined + resolveTools && context + ? await bindSupervisorTools(resolveTools, context, scope.signal) + : undefined const onEvent = bindSupervisorNodeObserver(context, observeNodeEvent, deps.onEvent) const mcp = await serveCoordinationMcp({ scope, @@ -398,11 +411,16 @@ function supervisorNodeContext( async function bindSupervisorTools( resolveTools: ResolveSupervisorTools, context: SupervisorNodeContext, + signal: AbortSignal, ): Promise> { const resolved = await resolveTools(context) if (!Array.isArray(resolved)) { throw new ValidationError('supervisorAgent: resolveSupervisorTools must return an array') } + // Keep the durable node snapshot free of live process objects. The invocation wrapper is frozen + // shallowly so handlers cannot replace identity or signal, while the AbortSignal itself remains + // live and follows the manager scope's existing root/parent cascade. + const invocationContext: SupervisorToolInvocationContext = Object.freeze({ ...context, signal }) const names = new Set(coordinationVerbNames) return Object.freeze( resolved.map((rawTool, index) => { @@ -441,7 +459,7 @@ async function bindSupervisorTools( handler: (raw: unknown) => handler( detachedSnapshot(raw, `supervisorAgent tool ${JSON.stringify(name)} input`), - context, + invocationContext, ), }) }), diff --git a/tests/kernel/supervisor-agent.test.ts b/tests/kernel/supervisor-agent.test.ts index 8bfaaa9b..c4fccf45 100644 --- a/tests/kernel/supervisor-agent.test.ts +++ b/tests/kernel/supervisor-agent.test.ts @@ -1,8 +1,9 @@ import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' +import { driverChild, withDriverExecutor } from '../../src/runtime/supervise/driver-executor' import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' -import { createSupervisor } from '../../src/runtime/supervise/supervisor' +import { createRootHandle, createSupervisor } from '../../src/runtime/supervise/supervisor' import { type DriveHarness, type ResolveSupervisorTools, @@ -288,9 +289,205 @@ describe('supervisorAgent — the brain is resolved from profile.harness (backen }) expect(Object.isFrozen(call.context)).toBe(true) expect(Object.isFrozen((call.context as { identity: unknown }).identity)).toBe(true) + expect((call.context as { signal: AbortSignal }).signal).toBeInstanceOf(AbortSignal) + expect((call.context as { signal: AbortSignal }).signal.aborted).toBe(false) } }) + it('RootHandle.abort cancels a product tool inside a recursive router manager', async () => { + const blobs = new InMemoryResultBlobStore() + const journal = new InMemorySpawnJournal() + const handle = createRootHandle() + let toolStarted!: () => void + const started = new Promise((resolve) => { + toolStarted = resolve + }) + let toolCancelled!: () => void + const cancelled = new Promise((resolve) => { + toolCancelled = resolve + }) + let nestedSignal: AbortSignal | undefined + const nested = supervisorAgent( + { name: 'nested-manager', harness: 'cli-base' }, + { + brain: scriptedBrain([ + { toolCalls: [{ name: 'run_experiment', arguments: { candidate: 'a' } }] }, + { content: 'must not continue after cancellation' }, + ]), + blobs, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + nodeContext: { + runId: 'recursive-tool-abort', + runNamespace: 'recursive-tool-abort-namespace', + ownerId: 'owner-nested', + depth: 1, + assignmentId: 'nested-assignment', + identity: { + profileDigest: `sha256:${'e'.repeat(64)}`, + taskDigest: `sha256:${'f'.repeat(64)}`, + }, + }, + resolveSupervisorTools: async () => [ + { + name: 'run_experiment', + description: 'Run a long product-owned experiment', + inputSchema: { type: 'object' }, + handler: async (_raw, context) => { + nestedSignal = context.signal + toolStarted() + await new Promise((_resolve, reject) => { + const onAbort = () => { + toolCancelled() + reject(new DOMException(String(context.signal.reason), 'AbortError')) + } + if (context.signal.aborted) onAbort() + else context.signal.addEventListener('abort', onAbort, { once: true }) + }) + return { unreachable: true } + }, + }, + ], + }, + ) + const root: Agent = { + name: 'root', + async act(task, scope) { + const spawned = scope.spawn( + driverChild( + { + name: 'nested-manager', + harness: 'cli-base', + metadata: { role: 'driver' }, + }, + nested, + journal, + ), + task, + { + budget: { maxIterations: 20, maxTokens: 20_000 }, + label: 'nested-manager', + }, + ) + if (!spawned.ok) throw new Error(spawned.reason) + await scope.next() + return undefined + }, + } + const supervisor = createSupervisor() + supervisor.attach(handle) + const running = supervisor.run(root, 'run the nested experiment', { + budget: { maxIterations: 100, maxTokens: 100_000 }, + runId: 'recursive-tool-abort', + journal, + blobs, + executors: withDriverExecutor(createExecutorRegistry()), + maxDepth: 4, + now: () => 0, + }) + + await started + handle.abort('stop the experiment tree') + await cancelled + const result = await running + + expect(result).toMatchObject({ kind: 'no-winner', reason: 'aborted' }) + expect(nestedSignal?.aborted).toBe(true) + expect(nestedSignal?.reason).toBe('stop the experiment tree') + }) + + it('a caller abort cancels a product tool invoked through the external MCP path', async () => { + const blobs = new InMemoryResultBlobStore() + const journal = new InMemorySpawnJournal() + const caller = new AbortController() + let toolStarted!: () => void + const started = new Promise((resolve) => { + toolStarted = resolve + }) + let toolCancelled!: () => void + const cancelled = new Promise((resolve) => { + toolCancelled = resolve + }) + let harnessFinished!: () => void + const finished = new Promise((resolve) => { + harnessFinished = resolve + }) + let externalSignal: AbortSignal | undefined + let externalResponse: unknown + const driveHarness: DriveHarness = async ({ coordinationMcpUrl }) => { + try { + externalResponse = await jsonRpc(coordinationMcpUrl, 'tools/call', { + name: 'run_experiment', + arguments: { candidate: 'b' }, + }) + } finally { + harnessFinished() + } + } + const root = supervisorAgent( + { name: 'external-manager', harness: 'opencode' }, + { + blobs, + makeWorkerAgent: () => deliveringLeaf('unused', {}), + perWorker, + driveHarness, + nodeContext: { + runId: 'external-tool-abort', + runNamespace: 'external-tool-abort-namespace', + ownerId: 'owner-external', + depth: 0, + identity: { + profileDigest: `sha256:${'1'.repeat(64)}`, + taskDigest: `sha256:${'2'.repeat(64)}`, + }, + }, + resolveSupervisorTools: async () => [ + { + name: 'run_experiment', + description: 'Run a long product-owned experiment', + inputSchema: { type: 'object' }, + handler: async (_raw, context) => { + externalSignal = context.signal + toolStarted() + await new Promise((_resolve, reject) => { + const onAbort = () => { + toolCancelled() + reject(new DOMException(String(context.signal.reason), 'AbortError')) + } + if (context.signal.aborted) onAbort() + else context.signal.addEventListener('abort', onAbort, { once: true }) + }) + return { unreachable: true } + }, + }, + ], + }, + ) + const running = createSupervisor().run(root, 'run the external experiment', { + budget: { maxIterations: 100, maxTokens: 100_000 }, + runId: 'external-tool-abort', + journal, + blobs, + executors: createExecutorRegistry(), + maxDepth: 4, + now: () => 0, + signal: caller.signal, + }) + + await started + caller.abort() + await cancelled + const result = await running + await finished + + expect(result).toMatchObject({ kind: 'no-winner', reason: 'aborted' }) + expect(externalSignal?.aborted).toBe(true) + expect(externalSignal?.reason).toBe('caller signal aborted') + expect(externalResponse).toMatchObject({ + error: { code: -32000, message: 'caller signal aborted' }, + }) + }) + it('captures the resolver and rejects descriptor collisions before brain compute or MCP listen', async () => { const seed = { runId: 'sup',