From fc4cb300552ad9dbf9ee2c06d43287808b4b98a3 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 00:48:38 -0600 Subject: [PATCH] fix(promotion): report the interval used for decisions --- .github/workflows/ci.yml | 4 +- .github/workflows/publish.yml | 4 +- CHANGELOG.md | 4 + bench/CHANGELOG.md | 4 + bench/package.json | 2 +- .../wait-for-published-dependencies.test.mjs | 8 +- docs/api/primitive-catalog.md | 18 +++-- docs/api/runtime.md | 9 ++- docs/canonical-api.md | 4 +- package.json | 4 +- pnpm-lock.yaml | 30 +++---- pnpm-workspace.yaml | 4 +- src/runtime/promotion-gate.ts | 30 +++---- .../fixtures/agent-improvement-proposal.json | 10 +-- .../agent-profile-improvement-proposal.json | 4 +- tests/kernel/strategy-evolution.test.ts | 28 +++++-- tests/kernel/strategy-suite.test.ts | 80 ++++++++++++++----- 17 files changed, 159 insertions(+), 88 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d048fe1..9f401abe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,14 +77,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-eval - ref: 8e4d6c95ca5beb6fc62a7c3c52c9fb581b928795 # v0.135.1 + ref: 1243ca2fa7ac9f91ada7c85b71b9f216f1b3f4ec # v0.135.2 path: .cohort/agent-eval persist-credentials: false - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-knowledge - ref: 502ef4954c0dc85eb0f629bfb6c062b3508977ef # v6.1.10 + ref: 47d046bdafdcd292af8104e84107c48f5a80f505 # v6.1.11 path: .cohort/agent-knowledge persist-credentials: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f9c9c9a8..33355bb6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -143,7 +143,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-eval - ref: 8e4d6c95ca5beb6fc62a7c3c52c9fb581b928795 # v0.135.1 + ref: 1243ca2fa7ac9f91ada7c85b71b9f216f1b3f4ec # v0.135.2 path: .cohort/agent-eval persist-credentials: false @@ -151,7 +151,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-knowledge - ref: 502ef4954c0dc85eb0f629bfb6c062b3508977ef # v6.1.10 + ref: 47d046bdafdcd292af8104e84107c48f5a80f505 # v6.1.11 path: .cohort/agent-knowledge persist-credentials: false diff --git a/CHANGELOG.md b/CHANGELOG.md index ce43c8dc..b7b60c69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.109.2 + +- Align Runtime with Eval 0.135.2 and Knowledge 6.1.11 so every improvement path uses the corrected paired promotion decisions. + ## 0.109.1 - Move `runToolLoop` and `streamToolLoop` to the Worker-safe `@tangle-network/agent-runtime/tool-loop` entrypoint. diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index 51367275..59a1a966 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.4.9 + +- Consume Runtime 0.109.2 with Eval 0.135.2 and Knowledge 6.1.11 so benchmark runs use the corrected paired promotion decisions. + ## 0.4.8 - Consume Runtime 0.109.0 through the canonical `./kernel` entrypoint. diff --git a/bench/package.json b/bench/package.json index b0b68af9..e17f699f 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.4.8", + "version": "0.4.9", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/bench/scripts/wait-for-published-dependencies.test.mjs b/bench/scripts/wait-for-published-dependencies.test.mjs index d65e98b9..459beb45 100644 --- a/bench/scripts/wait-for-published-dependencies.test.mjs +++ b/bench/scripts/wait-for-published-dependencies.test.mjs @@ -10,8 +10,8 @@ test('collects required published Tangle dependencies from a packed manifest', ( const dependencies = collectRequiredTangleDependencies({ name: '@tangle-network/agent-bench', dependencies: { - '@tangle-network/agent-runtime': '0.109.0', - '@tangle-network/agent-eval': '0.135.1', + '@tangle-network/agent-runtime': '0.109.2', + '@tangle-network/agent-eval': '0.135.2', undici: '^7.0.0', }, peerDependencies: { @@ -27,9 +27,9 @@ test('collects required published Tangle dependencies from a packed manifest', ( }) assert.deepEqual(dependencies, [ - { name: '@tangle-network/agent-eval', spec: '0.135.1' }, + { name: '@tangle-network/agent-eval', spec: '0.135.2' }, { name: '@tangle-network/agent-interface', spec: '>=0.36.0 <0.37.0' }, - { name: '@tangle-network/agent-runtime', spec: '0.109.0' }, + { name: '@tangle-network/agent-runtime', spec: '0.109.2' }, ]) }) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 5c52773a..4fd1de05 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.109.1` and `@tangle-network/agent-eval@0.135.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.109.2` and `@tangle-network/agent-eval@0.135.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -605,7 +605,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 594 exports. | `printBenchmarkReport` | function | Pretty-print a report — the "free optimization" verdict, with the cost vector. | | `probeSandboxCapabilities` | function | Probe (and memoize per client) what the loop may rely on. A client without a | | `profileRichnessFinding` | function | Turn a {@link ProfileRichness} verdict into a bus-routable `AnalystFinding` (area `profile-quality`). | -| `promotionGate` | function | Statistical promotion decision over a holdout benchmark: a seeded paired bootstrap (`heldoutSignificance`) whose CI lower bound must clear `deltaThreshold`. | +| `promotionGate` | function | Statistical promotion decision over a holdout benchmark using the outcome-appropriate interval selected by `heldoutSignificance`. | | `providerAsExecutor` | function | Adapt an environment provider into an `ExecutorFactory` for `createExecutor`. | | `providerAsSandboxClient` | function | Adapt a neutral environment provider to the `SandboxClient` interface used by existing loop paths. | | `queueOf` | function | Convenience: a `DispatchUnit` factory over a fixed array of tasks, for the common case where | @@ -1370,7 +1370,7 @@ Import from `@tangle-network/agent-eval` — 10 exports. ### STATISTICS — significance, intervals, effect size -Import from `@tangle-network/agent-eval` — 53 exports. +Import from `@tangle-network/agent-eval` — 61 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1389,12 +1389,17 @@ Import from `@tangle-network/agent-eval` — 53 exports. | `mcnemarPower` | function | Power of a McNemar test at a given number of paired observations, the inverse | | `mcnemarRequiredN` | function | Number of paired observations needed for a McNemar test to reach a target | | `mulberry32` | function | Tiny seedable PRNG (mulberry32) — deterministic resampling/shuffling, not | +| `pairedBinaryScale` | function | The common positive level `s` such that EVERY value across both paired arms is | | `pairedBootstrap` | function | Paired bootstrap on (after − before) deltas. Returns a CI on the chosen | | `pairedCohensDz` | function | Cohen's dz for paired observations: mean(after - before) divided by the | +| `pairedDecisionShape` | function | Which estimator {@link decidePairedPromotion} would use on this data, and the | | `pairedDeltaTest` | function | Tests whether a paired candidate-minus-baseline delta clears a threshold. | +| `pairedDeltaTieFraction` | function | Fraction of paired observations whose delta is an exact tie (\|after − before\| | | `pairedEvalueSequence` | function | Run the paired e-value sequence over an in-order delta stream. | | `pairedMde` | function | Minimum detectable paired effect (standardised units) for a target paired | | `pairedRiskDifference` | function | Paired risk difference (the effect-size companion to {@link mcnemar}): the | +| `pairedRiskDifferenceExact` | function | Paired risk difference with the EXACT CONDITIONAL interval — the estimator a | +| `pairedRiskDifferenceScore` | function | Paired risk difference with TANGO'S (1998) SCORE INTERVAL — the estimator a | | `pairedSignTest` | function | Exact one-sided sign test over paired differences. | | `pairedTTest` | function | Paired t-test — before/after measurements on the SAME items. | | `partialCredit` | function | Partial credit: returns 0-1 ratio of current toward target | @@ -1408,9 +1413,12 @@ Import from `@tangle-network/agent-eval` — 53 exports. | `wilcoxonSignedRank` | function | Wilcoxon signed-rank — paired, no distributional assumption on the deltas. | | `wilson` | function | Wilson score interval for a binomial proportion. Correct at small n and near | | `normalizeScores` | const | Identity: dimensions already follow "higher = better" by prompt convention | +| `ExactRiskDifferenceResult` | interface | A paired binary effect size with an EXACT interval and the exact test that | | `McNemarResult` | interface | Result of a McNemar paired-binary significance test. | +| `PairedMcNemarEvidence` | interface | McNemar's exact paired-binary evidence, on the two-point path only. | | `ProportionInterval` | interface | A binomial proportion estimate with a confidence interval. | | `RiskDifferenceResult` | interface | A paired binary effect size (treatment rate − control rate) with a CI. | +| `ScoreRiskDifferenceResult` | interface | A paired binary effect size with an interval that is valid at a NONZERO | **Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BootstrapOptions`, `BootstrapResult`, `ClusterBootstrapInterval`, `CorpusAgreementOptions`, `CorpusAgreementPerDimension`, `CorpusAgreementReport`, `CorpusScoreRecord`, `EProcess`, `EProcessOptions`, `EProcessState`, `EProcessStep`, `PairedBootstrapOptions`, `PairedBootstrapResult`, `WeightedCompositeInput`, `WeightedCompositeResult`, `CliffsMagnitude`. @@ -1455,8 +1463,8 @@ Import from `@tangle-network/agent-eval/campaign` — 329 exports. | `fsCampaignStorage` | function | Node-filesystem storage — the default. Lazily requires `node:fs` so the | | `gepaOptimizationMethod` | function | Turn an optional GEPA installation into an `OptimizationMethod`. | | `gitWorktreeAdapter` | function | Git-backed `WorktreeAdapter`: creates isolated worktrees on fresh branches, commits agent changes, and discards losers. | -| `heldOutGate` | function | Composable held-out gate: ships only when the PAIRED bootstrap CI lower bound | -| `heldoutSignificance` | function | Significance of the held-out composite lift: ship only when the paired | +| `heldOutGate` | function | Composable held-out gate: ships only when the lower bound of the DECIDING | +| `heldoutSignificance` | function | Significance of the held-out composite lift: ship only when the lower bound | | `inMemoryCampaignStorage` | function | In-memory storage for filesystem-less runtimes. Artifacts + trace spans | | `isProposedCandidate` | function | Type guard: a proposal carrying its rationale vs a bare | | `isTransientTransportFailure` | function | True when the error text describes an infrastructure hiccup that should be | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index a20caaeb..7bbea49f 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -4923,7 +4923,8 @@ Paired tasks that carried both strategies' cells. > **lift**: `object` -Paired (candidate − incumbent) lift across the holdout tasks. +Paired (candidate − incumbent) lift across the holdout tasks. `low` and `high` + are the bounds that carried the decision; `mean` and `median` are diagnostics. ###### mean @@ -4945,8 +4946,8 @@ Paired (candidate − incumbent) lift across the holdout tasks. > `optional` **costSavings?**: `object` -non-inferiority mode: paired (incumbent − candidate) cost SAVINGS per task (usd) — - positive means the candidate is cheaper; significant iff the CI low clears zero. +non-inferiority mode: paired (incumbent − candidate) cost savings per task (usd). + Positive means the candidate is cheaper; `low` and `high` carried the decision. ###### mean @@ -18282,7 +18283,7 @@ readonly [`EqualKArm`](#equalkarm)[] > **promotionGate**(`opts`): [`PromotionVerdict`](#promotionverdict) -Statistical promotion decision over a holdout benchmark: a seeded paired bootstrap (`heldoutSignificance`) whose CI lower bound must clear `deltaThreshold`. +Statistical promotion decision over a holdout benchmark using the outcome-appropriate interval selected by `heldoutSignificance`. #### Parameters diff --git a/docs/canonical-api.md b/docs/canonical-api.md index b9cf8943..8afa001e 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,9 +4,9 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.109.1.** +> **Version 0.109.2.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> `agent-eval` must satisfy `>=0.135.1 <0.136.0`. +> `agent-eval` must satisfy `>=0.135.2 <0.136.0`. > `sandbox` must satisfy `>=0.15.0 <0.16.0`. > Portable profile and tool-part types come from `@tangle-network/agent-interface` `>=0.36.0 <0.37.0`. > diff --git a/package.json b/package.json index abff23ab..1361e708 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.109.1", + "version": "0.109.2", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -162,7 +162,7 @@ "license": "MIT", "packageManager": "pnpm@11.17.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.135.1 <0.136.0", + "@tangle-network/agent-eval": ">=0.135.2 <0.136.0", "@tangle-network/agent-interface": ">=0.36.0 <0.37.0", "@tangle-network/sandbox": ">=0.15.0 <0.16.0", "playwright": "^1.40.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2cd682b1..53ca2385 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,14 +10,14 @@ catalogs: specifier: 0.18.5 version: 0.18.5 '@tangle-network/agent-eval': - specifier: 0.135.1 - version: 0.135.1 + specifier: 0.135.2 + version: 0.135.2 '@tangle-network/agent-interface': specifier: 0.36.0 version: 0.36.0 '@tangle-network/agent-knowledge': - specifier: 6.1.10 - version: 6.1.10 + specifier: 6.1.11 + version: 6.1.11 '@tangle-network/agent-profile-materialize': specifier: 0.9.2 version: 0.9.2 @@ -46,7 +46,7 @@ importers: dependencies: '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 6.1.10 + version: 6.1.11 '@tangle-network/agent-profile-materialize': specifier: 'catalog:' version: 0.9.2(@tangle-network/agent-interface@0.36.0) @@ -62,7 +62,7 @@ importers: version: 2.5.5 '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.135.1 + version: 0.135.2 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.36.0 @@ -113,13 +113,13 @@ importers: dependencies: '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.135.1 + version: 0.135.2 '@tangle-network/agent-interface': specifier: 'catalog:' version: 0.36.0 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 6.1.10 + version: 6.1.11 '@tangle-network/agent-runtime': specifier: workspace:* version: link:.. @@ -1111,16 +1111,16 @@ packages: '@tangle-network/agent-core@0.4.25': resolution: {integrity: sha512-3hfIs64b/1/C1ezYhSPu6NC449lO4XAq79uaol0z0eLXcXDM+2znXThTeMTEZUPJ4iAsR514xcHL2OrP3dsE3g==} - '@tangle-network/agent-eval@0.135.1': - resolution: {integrity: sha512-YKJEn4+IJDSNYTP9MQTDqcyRIxcSVetkqxYqv+IWEPfVM9odsAQ95tmatb5wlYukuz9u5/Dk7Dm5AAFrHKZW6A==} + '@tangle-network/agent-eval@0.135.2': + resolution: {integrity: sha512-qgthVcu8gQy57SgKFnx+Oug0t3ooPoqsDEMteNdye+EbkcvDH7k42JJrR4AYDRW0mbfNsA3WdzhlaGAK+Dd+Ew==} engines: {node: '>=20'} hasBin: true '@tangle-network/agent-interface@0.36.0': resolution: {integrity: sha512-l48SiAY5Atx6ZTL86/XP2wSUPfJJ1sga82/F6V0KqwVkE5MHWiFql6wsLoqk0lc7s4EJe6+cKky+2zzxFuxDpg==} - '@tangle-network/agent-knowledge@6.1.10': - resolution: {integrity: sha512-M+3kmoG/wjs18TY6I7XjyvmZB1GHIdO0MwiMefSlOxrPm6+i6D8sqd5TLwh0xUt52FuDAHfW2je3jUlAI7S4hA==} + '@tangle-network/agent-knowledge@6.1.11': + resolution: {integrity: sha512-2p5TLMreGGEsYp+V2LWVm1YjTfLlHHaq5cT1qtLmN4Ao7Ku5pnCdptq6sZwf8kbgHUAwPWFuomWw7iXTzKMeiw==} engines: {node: '>=20.19.0'} hasBin: true @@ -2757,7 +2757,7 @@ snapshots: '@tangle-network/agent-interface': 0.36.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.135.1': + '@tangle-network/agent-eval@0.135.2': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) '@ax-llm/ax': 23.0.5(zod@4.4.3) @@ -2772,9 +2772,9 @@ snapshots: '@noble/hashes': 1.8.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@6.1.10': + '@tangle-network/agent-knowledge@6.1.11': dependencies: - '@tangle-network/agent-eval': 0.135.1 + '@tangle-network/agent-eval': 0.135.2 '@tangle-network/agent-interface': 0.36.0 proper-lockfile: 4.1.2 zod: 4.4.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a5ecb70c..4b61b71a 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -18,9 +18,9 @@ allowBuilds: catalog: '@arethetypeswrong/cli': 0.18.5 '@types/node': 26.1.1 - '@tangle-network/agent-eval': 0.135.1 + '@tangle-network/agent-eval': 0.135.2 '@tangle-network/agent-interface': 0.36.0 - '@tangle-network/agent-knowledge': 6.1.10 + '@tangle-network/agent-knowledge': 6.1.11 '@tangle-network/agent-profile-materialize': 0.9.2 '@tangle-network/sandbox': 0.15.2 publint: 0.3.22 diff --git a/src/runtime/promotion-gate.ts b/src/runtime/promotion-gate.ts index 8e9b80ea..4ce2a5fc 100644 --- a/src/runtime/promotion-gate.ts +++ b/src/runtime/promotion-gate.ts @@ -1,11 +1,10 @@ /** * promotionGate — the statistical promotion decision over a holdout benchmark: does the * candidate strategy beat the incumbent on held-out tasks by a margin the task noise - * cannot fake? The statistics are the substrate's (`heldoutSignificance`): a SEEDED - * paired bootstrap over per-task (candidate − incumbent) deltas — deterministic verdict, - * a minimum-evidence floor, and the CI lower bound must clear `deltaThreshold`. A raw - * h1>h0 point comparison on m≈8 holdout tasks certifies false champions at near - * coin-flip rates; this gate is the instrument-grade replacement. + * cannot fake? The statistics come from `heldoutSignificance`, which chooses the + * decision interval that matches the outcome shape, applies a minimum-evidence floor, + * and requires the interval's lower bound to clear `deltaThreshold`. A raw h1>h0 point + * comparison on m≈8 holdout tasks certifies false champions at near coin-flip rates. */ import { heldoutSignificance } from '@tangle-network/agent-eval/campaign' import type { BenchmarkReport } from './run-benchmark' @@ -49,10 +48,11 @@ export interface PromotionVerdict { mode: 'superiority' | 'non-inferiority' /** Paired tasks that carried both strategies' cells. */ n: number - /** Paired (candidate − incumbent) lift across the holdout tasks. */ + /** Paired (candidate − incumbent) lift across the holdout tasks. `low` and `high` + * are the bounds that carried the decision; `mean` and `median` are diagnostics. */ lift: { mean: number; median: number; low: number; high: number } - /** non-inferiority mode: paired (incumbent − candidate) cost SAVINGS per task (usd) — - * positive means the candidate is cheaper; significant iff the CI low clears zero. */ + /** non-inferiority mode: paired (incumbent − candidate) cost savings per task (usd). + * Positive means the candidate is cheaper; `low` and `high` carried the decision. */ costSavings?: { mean: number; median: number; low: number; high: number } /** Paired (candidate − incumbent) wall-clock per task (ms) — negative = the candidate * is FASTER. Informational in every mode (never gates); the latency answer to "what @@ -60,7 +60,7 @@ export interface PromotionVerdict { latency?: { mean: number; median: number; low: number; high: number } } -/** Statistical promotion decision over a holdout benchmark: a seeded paired bootstrap (`heldoutSignificance`) whose CI lower bound must clear `deltaThreshold`. */ +/** Statistical promotion decision over a holdout benchmark using the outcome-appropriate interval selected by `heldoutSignificance`. */ export function promotionGate(opts: PromotionGateOptions): PromotionVerdict { const mode = opts.mode ?? 'superiority' if (opts.candidate === opts.incumbent) { @@ -109,8 +109,8 @@ export function promotionGate(opts: PromotionGateOptions): PromotionVerdict { const lift = { mean: sig.bootstrap.mean, median: sig.bootstrap.median, - low: sig.bootstrap.low, - high: sig.bootstrap.high, + low: sig.decision.low, + high: sig.decision.high, } const latSig = heldoutSignificance( { before: incMs, after: candMs, cellIds }, @@ -125,8 +125,8 @@ export function promotionGate(opts: PromotionGateOptions): PromotionVerdict { const latency = { mean: latSig.bootstrap.mean, median: latSig.bootstrap.median, - low: latSig.bootstrap.low, - high: latSig.bootstrap.high, + low: latSig.decision.low, + high: latSig.decision.high, } if (mode === 'superiority') { if (sig.fewRuns) return { promoted: false, reason: 'few-tasks', mode, n: sig.n, lift, latency } @@ -160,8 +160,8 @@ export function promotionGate(opts: PromotionGateOptions): PromotionVerdict { const costSavings = { mean: costSig.bootstrap.mean, median: costSig.bootstrap.median, - low: costSig.bootstrap.low, - high: costSig.bootstrap.high, + low: costSig.decision.low, + high: costSig.decision.high, } if (scoreSig.fewRuns) return { promoted: false, reason: 'few-tasks', mode, n: scoreSig.n, lift, costSavings, latency } diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 516bf670..86788a8b 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:218d8ea27510e757b998843e63900712a5b12027484d4a481f29e44f016c8307", + "digest": "sha256:fc52fd656a37a79611a2f870079695fe1a5c055d3078d58b0f2d4c79d73c3945", "evaluation": { "decision": { "contributingChecks": [ @@ -4810,7 +4810,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.109.1" + "runtimeVersion": "0.109.2" }, "objectives": [ { @@ -4921,8 +4921,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:8ca3b2249f293c547e35cfe3cc79e041dff9cf29a3be5a1e2850fd157e719a51", - "runId": "agent-runtime-0.109.1-proposal-fixture", + "recordDigest": "sha256:042b806bcb6e3db2fe3b8744d543d769a0d3edf6cebe5413becb137196b00fd8", + "runId": "agent-runtime-0.109.2-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -4949,5 +4949,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.109.1-proposal-fixture" + "runId": "agent-runtime-0.109.2-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 0ad78f00..61f9ef3d 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:34e61cc608be4952ec34dce30fcf94fbec0d39018373913c72a7fdbbf20aaec4", + "digest": "sha256:5a6888221983315e661a139d32348538dc054443e03ce2ef7f0cdf6380fb3558", "evaluation": { "decision": { "contributingChecks": [ @@ -1695,7 +1695,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.109.1" + "runtimeVersion": "0.109.2" }, "objectives": [ { diff --git a/tests/kernel/strategy-evolution.test.ts b/tests/kernel/strategy-evolution.test.ts index 0e419511..281ebab8 100644 --- a/tests/kernel/strategy-evolution.test.ts +++ b/tests/kernel/strategy-evolution.test.ts @@ -20,18 +20,26 @@ import { runStrategyEvolution, selectChampion } from '../../src/runtime/strategy // ── Fixtures ────────────────────────────────────────────────────────────────────── -/** Deterministic surface: score = shots taken on the handle, capped at 2 of 2 — one - * worker pass per shot is observable via tools() calls. Depth (2 shots, one handle) - * scores 1.0; breadth (fresh handle per shot) scores 0.5. */ +const taskCheckTotal = (taskId: string | undefined): number => { + const index = Number(taskId?.match(/\d+$/)?.[0] ?? 0) + return 2 + (index % 3) +} + +/** Deterministic surface with task difficulty varying from 2 to 4 checks. One worker + * pass per shot is observable via tools() calls. Depth takes two shots on one handle; + * breadth takes one shot on each fresh handle, so every paired task favors depth while + * producing varied holdout gains. */ function shotCountingSurface(): AgenticSurface { const shotsByHandle = new Map() + const taskByHandle = new Map() let seq = 0 return { name: 'shot-counter', - async open() { + async open(task) { seq += 1 const id = `h-${seq}` shotsByHandle.set(id, 0) + taskByHandle.set(id, task.id) return { id, surface: 'shot-counter' } }, async tools(_t, handle) { @@ -42,7 +50,11 @@ function shotCountingSurface(): AgenticSurface { return 'ok' }, async score(_t, handle) { - return { passes: Math.min(shotsByHandle.get(handle.id) ?? 0, 2), total: 2, errored: 0 } + return { + passes: Math.min(shotsByHandle.get(handle.id) ?? 0, 2), + total: taskCheckTotal(taskByHandle.get(handle.id)), + errored: 0, + } }, async close() {}, } @@ -319,7 +331,11 @@ function difficultySurface(): AgenticSurface { async score(_t, handle) { if (taskByHandle.get(handle.id)?.startsWith('easy-')) return { passes: 2, total: 2, errored: 0 } - return { passes: Math.min(shotsByHandle.get(handle.id) ?? 0, 2), total: 2, errored: 0 } + return { + passes: Math.min(shotsByHandle.get(handle.id) ?? 0, 2), + total: taskCheckTotal(taskByHandle.get(handle.id)), + errored: 0, + } }, async close() {}, } diff --git a/tests/kernel/strategy-suite.test.ts b/tests/kernel/strategy-suite.test.ts index 0288a7f3..63da6ffb 100644 --- a/tests/kernel/strategy-suite.test.ts +++ b/tests/kernel/strategy-suite.test.ts @@ -13,6 +13,7 @@ import { mkdtempSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { heldoutSignificance } from '@tangle-network/agent-eval/campaign' import { afterEach, describe, expect, it, vi } from 'vitest' import { InMemoryCorpus } from '../../src/runtime/personify/corpus' import { promotionGate } from '../../src/runtime/promotion-gate' @@ -387,6 +388,25 @@ describe('promotionGate', () => { expect(b).toEqual(a) }) + it('reports the paired binary decision interval instead of bootstrap diagnostics', () => { + const before = Array(20).fill(0) + const after = Array.from({ length: 20 }, (_, i) => (i < 12 ? 1 : 0)) + const cellIds = before.map((_, i) => `t${i}`) + const report = reportWith(cellIds.map((id, i) => ({ id, inc: before[i]!, cand: after[i]! }))) + const significance = heldoutSignificance({ before, after, cellIds }) + const verdict = promotionGate({ + report, + incumbent: 'incumbent', + candidate: 'candidate', + }) + + expect(verdict.promoted).toBe(true) + expect(verdict.lift.low).toBe(significance.decision.low) + expect(verdict.lift.high).toBe(significance.decision.high) + expect(verdict.lift.low).not.toBe(significance.bootstrap.low) + expect(verdict.lift.high).not.toBe(significance.bootstrap.high) + }) + it('symmetric noise does not promote (CI includes zero)', () => { const rows = Array.from({ length: 12 }, (_, i) => ({ id: `t${i}`, @@ -655,13 +675,19 @@ describe('promotionGate non-inferiority', () => { }) it('same quality at half the cost PROMOTES', () => { - const rows = Array.from({ length: 12 }, (_, i) => ({ - id: `t${i}`, - incScore: 0.6 + (i % 3) * 0.05, - candScore: 0.59 + (i % 3) * 0.05, - incUsd: 0.028, - candUsd: 0.013, - })) + const scoreDeltas = [-0.018, -0.012, -0.008, -0.004, 0, 0.006, 0.011, -0.006] + const costSavings = [0.011, 0.014, 0.016, 0.013, 0.018, 0.015, 0.012, 0.017] + const rows = Array.from({ length: 24 }, (_, i) => { + const incScore = 0.48 + (i % 7) * 0.055 + const incUsd = 0.024 + (i % 5) * 0.0014 + return { + id: `t${i}`, + incScore, + candScore: incScore + scoreDeltas[i % scoreDeltas.length]!, + incUsd, + candUsd: incUsd - costSavings[i % costSavings.length]!, + } + }) const v = promotionGate({ report: costReport(rows), incumbent: 'incumbent', @@ -693,13 +719,19 @@ describe('promotionGate non-inferiority', () => { }) it('same quality at the same cost is NOT a promotion', () => { - const rows = Array.from({ length: 12 }, (_, i) => ({ - id: `t${i}`, - incScore: 0.6, - candScore: 0.61, - incUsd: 0.02, - candUsd: i % 2 === 0 ? 0.021 : 0.019, - })) + const scoreDeltas = [-0.006, -0.002, 0.003, 0.008, 0.012, 0.005] + const costSavings = [-0.0018, 0.0012, -0.0007, 0.0005, 0.0017, -0.0011] + const rows = Array.from({ length: 24 }, (_, i) => { + const incScore = 0.5 + (i % 6) * 0.06 + const incUsd = 0.018 + (i % 5) * 0.0012 + return { + id: `t${i}`, + incScore, + candScore: incScore + scoreDeltas[i % scoreDeltas.length]!, + incUsd, + candUsd: incUsd - costSavings[i % costSavings.length]!, + } + }) const v = promotionGate({ report: costReport(rows), incumbent: 'incumbent', @@ -711,13 +743,19 @@ describe('promotionGate non-inferiority', () => { }) it('the verdict is deterministic', () => { - const rows = Array.from({ length: 12 }, (_, i) => ({ - id: `t${i}`, - incScore: 0.6, - candScore: 0.6, - incUsd: 0.03, - candUsd: 0.012 + (i % 2) * 0.002, - })) + const scoreDeltas = [-0.01, -0.006, -0.002, 0.003, 0.007, 0.011] + const costSavings = [0.014, 0.016, 0.018, 0.013, 0.017, 0.015] + const rows = Array.from({ length: 24 }, (_, i) => { + const incScore = 0.47 + (i % 7) * 0.058 + const incUsd = 0.027 + (i % 5) * 0.0013 + return { + id: `t${i}`, + incScore, + candScore: incScore + scoreDeltas[i % scoreDeltas.length]!, + incUsd, + candUsd: incUsd - costSavings[i % costSavings.length]!, + } + }) const report = costReport(rows) const a = promotionGate({ report,