From d578ff15d3865624789744cedffad1a3fdc01d79 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 23 Jul 2026 02:40:30 -0700 Subject: [PATCH 1/8] feat(testing): add test methodology and report tooling --- REUSE.toml | 5 + docs/testing/dstack-test-methodology.md | 199 ++++ docs/testing/test-case-authoring-spec.md | 300 ++++++ docs/testing/test-report-output-spec.md | 274 +++++ tools/dstack-test/README.md | 81 ++ tools/dstack-test/dstack-test | 6 + tools/dstack-test/dstack_test.py | 633 ++++++++++++ tools/dstack-test/render.py | 932 ++++++++++++++++++ .../01-proxy-protocol/tc-gw-pp-001/case.md | 20 + .../tc-gw-pp-001/results/run-demo/SHA256SUMS | 5 + .../run-demo/artifacts/backend-capture.json | 1 + .../tc-gw-pp-001/results/run-demo/prompt.md | 1 + .../tc-gw-pp-001/results/run-demo/result.json | 14 + .../tc-gw-pp-001/results/run-demo/runner.json | 15 + .../results/run-demo/session.jsonl | 7 + .../tests/fixtures/sample-plan/README.md | 4 + .../tests/fixtures/sample-plan/index.json | 38 + .../sample-plan/results/run-demo/SHA256SUMS | 1 + .../sample-plan/results/run-demo/run.json | 31 + tools/dstack-test/tests/test_dstack_test.py | 149 +++ 20 files changed, 2716 insertions(+) create mode 100644 docs/testing/dstack-test-methodology.md create mode 100644 docs/testing/test-case-authoring-spec.md create mode 100644 docs/testing/test-report-output-spec.md create mode 100644 tools/dstack-test/README.md create mode 100755 tools/dstack-test/dstack-test create mode 100755 tools/dstack-test/dstack_test.py create mode 100755 tools/dstack-test/render.py create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/SHA256SUMS create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/artifacts/backend-capture.json create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/prompt.md create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/result.json create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/runner.json create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/session.jsonl create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/README.md create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/index.json create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/SHA256SUMS create mode 100644 tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/run.json create mode 100644 tools/dstack-test/tests/test_dstack_test.py diff --git a/REUSE.toml b/REUSE.toml index f68f2a21a..240cc30da 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -203,6 +203,11 @@ path = "**/src/generated/*" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" +[[annotations]] +path = "tools/dstack-test/tests/fixtures/**" +SPDX-FileCopyrightText = "NONE" +SPDX-License-Identifier = "CC0-1.0" + [[annotations]] path = "dstack/gateway/assets/*" SPDX-FileCopyrightText = "NONE" diff --git a/docs/testing/dstack-test-methodology.md b/docs/testing/dstack-test-methodology.md new file mode 100644 index 000000000..8a3c84c2c --- /dev/null +++ b/docs/testing/dstack-test-methodology.md @@ -0,0 +1,199 @@ + + + + +# dstack 测试方法 + +本文规定 dstack 版本测试从需求分析、风险建模、用例设计、环境执行、证据采集到发布判定的统一方法。具体用例格式见[测试用例编写规范](test-case-authoring-spec.md#dstack-test-case-authoring-spec)，执行产物格式见[测试报告输出规范](test-report-output-spec.md#dstack-test-report-output-spec)。 + + +## 1. 目标 + +测试应当提供可重复、可审计且可追踪的发布证据，而不只是证明某个脚本曾经退出为零。任一结论都应能从需求或风险追踪到用例、步骤、命令原文、观察结果和附件。 + +测试工作的完成条件是： + +1. 版本变化、产品需求和主要风险均有明确测试覆盖； +2. 测试规格可由不了解实现细节的执行者独立复现； +3. 每一步实际执行的命令、退出码、标准输出和标准错误均保存； +4. 模拟测试和真实硬件测试分别报告，不能互相替代； +5. 汇总状态可由工具从原子结果重新计算； +6. 报告中的引用、附件摘要和统计均可机器校验。 + + +## 2. 测试资料分层 + +测试资料分为四类，禁止混写： + +| 资料 | 作用 | 是否随执行变化 | +|---|---|---| +| 测试说明 `README.md` | 范围、环境配置、公共前置条件、执行与清理方法 | 否 | +| 测试索引 `index.json` | 章、节、用例顺序及机器可读引用 | 否 | +| 用例规格 `case.md` | 单条用例的目标、前置条件、数据、步骤和预期结果 | 否 | +| 执行结果 `results//` | 实际版本、命令证据、观察结果、附件和状态 | 是 | + +测试规格是只读输入。执行器只能在 `results/` 下创建当前 run 的产物，不得根据执行结果反向修改预期结果。 + + +## 3. 组织结构 + +测试计划固定使用“章／节／用例”三个层级。每条用例必须拥有独立目录，便于附带 fixture、脚本和多次执行结果。 + +```text +/ +├── index.json +├── README.md +├── schemas/ +├── results//run.json +└── / + └──
/ + └── / + ├── case.md + ├── fixtures/ # 可选 + ├── scripts/ # 可选 + └── results// +``` + +目录名建议使用两位数字前缀维持人工浏览顺序；权威执行顺序仍来自 `index.json`，执行器不得通过文件名猜测顺序。 + + +## 4. 测试输入分析 + +在编写用例前，测试负责人应收集并冻结： + +- 上一个发布版本及本次候选 revision； +- change log、PR、schema、RPC 和配置变化； +- 产品需求及用户场景； +- 安全不变量和信任边界； +- 已知缺陷、迁移约束和运维方式； +- 可用的真实硬件、模拟器与外部服务。 + +每项变化至少回答：谁使用它、正常行为是什么、错误输入如何处理、升级时如何兼容、失败后如何恢复、会影响哪些旧功能。 + + +## 5. 追踪与覆盖 + +每条用例应引用至少一个需求或风险。推荐维护以下关系： + +```text +Change / Requirement / Risk + ↓ + Test case + ↓ + Step + ↓ + Command and output + ↓ + Assertion/result +``` + +覆盖审计至少包含： + +1. 新增和改变的功能； +2. 受影响功能的回归； +3. 新旧组件组合兼容； +4. 配置和 API 的合法、边界及错误输入； +5. 有状态组件的创建、运行、停止、重启、升级、回滚和恢复； +6. 安全策略的允许、拒绝及恢复； +7. 并发、隔离、幂等和重试； +8. 资源耗尽、依赖不可用和网络故障； +9. 可观测性、错误信息及审计记录。 + +覆盖数量不是质量指标。一个包含多个独立断言的大用例应拆为若干原子用例，否则失败无法定位，结果也无法准确统计。 + + +## 6. 环境等级 + +执行报告必须标记环境等级： + +| 等级 | 说明 | +|---|---| +| `UNIT` | 单进程或模块测试 | +| `INTEGRATION` | 多组件集成，但不使用模拟硬件证据代替真实平台 | +| `SIMULATED` | mock attestation、no-TEE、swtpm 等模拟环境 | +| `REAL_SGX` | 真实 SGX enclave | +| `REAL_TDX` | 真实 TDX CVM | +| `REAL_PLATFORM` | 指定 NVIDIA、GCP、AWS、SEV-SNP 等真实平台 | + +用例规格必须声明最低环境。若只在较低等级执行，原用例保持 `NOT_RUN` 或 `BLOCKED`，模拟结果应记录在单独用例中。 + + +## 7. 状态与判定 + +状态枚举固定为： + +- `PASS`：所有步骤的实际结果完全满足预期； +- `FAIL`：至少一步实际结果不完全满足预期； +- `BLOCKED`：外部环境或前置条件阻止被测行为开始； +- `NOT_RUN`：尚未执行； +- `SKIPPED`：预先定义的适用条件不成立。 + +单个用例不使用 `PARTIAL`。若一部分断言通过、一部分未执行，应拆分用例或将该用例判为非 `PASS`。顶层汇总必须由用例结果计算，不能手工覆盖。 + + +## 8. 标准执行流程 + +AI或人工执行器应按以下顺序工作： + +1. 阅读顶层 `README.md`； +2. 读取并校验 `index.json`、schema、路径、锚点和引用； +3. 创建唯一 `run_id`，记录执行器、公共版本和环境； +4. 按索引顺序逐章、逐节、逐用例读取 `case.md`； +5. 为每条用例启动独立AI会话，验证前置条件并执行各步骤； +6. runner将Agent原生JSONL事件流直接保存为`session.jsonl`，其中包含实际tool call、命令和原始输出； +7. 将观察结果与预期逐项比较，结束前原子写入浅层`result.json`； +8. 执行用例后置条件，记录未能清理的资源； +9. 全部完成后重新校验会话、结果、附件摘要、引用和统计； +10. 生成顶层`run.json`、`SHA256SUMS`，再打包上传或渲染HTML。 + +AI执行器必须记录实际提供方、产品和模型名称；无法从运行环境确认模型时填写 `unknown`，不得猜测。 + + +## 9. 证据原则 + +每一步至少包含一条实际执行的命令证据。命令和输出由Agent原生 +`session.jsonl`保存，不要求AI再次复制或转义。会话应包含： + +- 未经改写的命令字符串和工作目录； +- 已脱敏的环境变量； +- UTC开始、结束时间和退出码； +- stdout、stderr原文； +- 从原文提取的结构化 observed 值； +- 截图、日志或二进制捕获附件及 SHA-256。 + +额外的截图、长日志或二进制捕获写入`artifacts/`并由`result.json`引用； +`result.json`只保存逐步观察总结，不嵌入大段输出。密钥、token、私钥和用户数据 +不得进入会话或附件；脱敏必须保留字段存在性和数据形状，且在报告中声明。 + + +## 10. 版本与环境记录 + +公共被测版本只在 run 顶层记录一次，包括 Git revision、binary SHA-256、container digest 和 guest image digest。单个用例只在使用旧版或特殊构件时设置 `version_overrides`。 + +环境记录至少包括：主机标识、内核、CPU架构、TEE能力、模拟标志、关键外部服务及配置摘要。路径和主机名如涉及隐私，可使用稳定别名，但必须足以复现拓扑。 + + +## 11. 发布判定 + +发布结论应直接引用未通过的 P0/P1 用例及未覆盖需求。以下情况不得表述为“全量通过”： + +- P0 用例为 `FAIL`、`BLOCKED` 或 `NOT_RUN`； +- 真实硬件要求仅由模拟测试覆盖； +- 关键附件缺失或摘要不匹配； +- 结果引用失效； +- 执行偏离规格且未记录 deviation； +- 公共版本未固定到可追溯 digest。 + + +## 12. 打包和展示 + +规范 Plan目录是平台和本地渲染器的唯一输入。推荐命令： + +```bash +dstack-test render \ + --plan path/to/plan \ + --run-id run-20260723-001 \ + --output report.html +``` + +渲染前必须完成结构、引用、状态、统计和附件摘要校验。生成的 HTML应内联样式、脚本、JSON、文本、图片和可下载附件，不依赖网络资源。 diff --git a/docs/testing/test-case-authoring-spec.md b/docs/testing/test-case-authoring-spec.md new file mode 100644 index 000000000..d728aaaba --- /dev/null +++ b/docs/testing/test-case-authoring-spec.md @@ -0,0 +1,300 @@ + + + + +# dstack 测试用例编写规范 + +本文规定测试 Plan、索引和 `case.md` 的写法。总体方法见[dstack 测试方法](dstack-test-methodology.md#dstack-test-methodology)，结果格式见[测试报告输出规范](test-report-output-spec.md#dstack-test-report-output-spec)。 + + +## 1. Plan目录 + +```text +/ +├── index.json +├── README.md +├── schemas/ +├── results/ +└── / + ├── README.md # 可选 + └──
/ + ├── README.md # 可选 + └── / + ├── case.md + ├── fixtures/ # 可选 + ├── scripts/ # 可选 + └── results/ +``` + +测试组织只允许章、节、用例三个语义层级。用例必须放在独立目录中，规格文件固定命名为 `case.md`。 + + +## 2. ID和锚点 + +所有可引用对象必须使用显式、稳定、全局唯一的 ASCII ID： + +```text +chapter-gateway +section-gateway-proxy-protocol +tc-gw-pp-001 +tc-gw-pp-001-step-01 +req-gw-pp-inbound-v1 +risk-gw-pp-client-address +``` + +规则： + +- 仅使用小写字母、数字和连字符； +- ID发布后不得因标题变化而改变； +- Markdown不得依赖标题自动生成的 slug； +- 每个可引用标题前必须写 ``； +- 跨文件引用使用相对路径和 fragment，例如 `case.md#tc-gw-pp-001-step-01`。 + + +## 3. 顶层说明书 + +顶层 `README.md`是执行器必须首先阅读的文件，至少包含： + +1. 测试目标和范围； +2. 被测系统和测试拓扑； +3. 硬件、软件、账号及外部服务要求； +4. 公共环境配置命令； +5. 公共前置条件和健康检查； +6. 状态判定规则； +7. 证据、脱敏和附件规则； +8. 执行顺序、并发限制和停止条件； +9. 清理与环境恢复方法； +10. 打包、上传和本地渲染方法。 + +环境配置命令应可直接执行；不得使用“配置好KMS”“准备可用证书”等无法复现的描述。 + + +## 4. `index.json` + +索引是用例发现和顺序的权威来源。最低结构如下： + +```json +{ + "schema_version": "1.0", + "id": "plan-gateway-proxy-protocol", + "title": "Gateway Proxy Protocol Test Plan", + "guide": { + "path": "README.md", + "anchor": "test-plan-guide" + }, + "chapters": [ + { + "id": "chapter-gateway", + "order": 1, + "title": "Gateway", + "path": "01-gateway", + "sections": [ + { + "id": "section-gateway-proxy-protocol", + "order": 1, + "title": "Proxy Protocol", + "path": "01-gateway/01-proxy-protocol", + "cases": [ + { + "id": "tc-gw-pp-001", + "order": 1, + "title": "TLS终止路径转发Proxy v1地址", + "priority": "P0", + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001", + "spec": { + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md", + "anchor": "tc-gw-pp-001" + }, + "requirements": ["req-gw-pp-inbound-v1"], + "risks": ["risk-gw-pp-client-address"], + "tags": ["gateway", "proxy-protocol", "e2e"] + } + ] + } + ] + } + ] +} +``` + +章、节和用例数组顺序必须与 `order`一致。路径必须留在 Plan根目录内，不得包含绝对路径或 `..`逃逸。 + + +## 5. `case.md`固定结构 + +每条用例按以下顺序编写： + +```markdown + +# TC-EXAMPLE-001：标题 + + +## 元数据 + + +## 测试目标 + + +## 前置条件 + + +## 测试数据 + + +## 测试步骤 + + +### Step 1：步骤标题 + +操作说明。 + +**预期结果：** + +- 可观察结果一； +- 可观察结果二。 + + +## 后置条件 +``` + + +## 6. 元数据 + +元数据至少包含： + +- ID； +- 优先级 `P0`、`P1`或`P2`； +- 类型，例如 Functional、Security、Compatibility、Regression、Performance； +- 最低环境等级； +- 是否适合自动化； +- Requirement和Risk引用； +- Tags。 + +公共版本不在每条用例重复。需要旧版或特殊组件时，在用例中增加“特殊版本要求”，执行结果写入 `version_overrides`。 + + +## 7. 测试目标 + +目标只描述一个可独立判定的行为。若标题中出现多个互不依赖的“并且”，通常应拆分用例。目标需要说明： + +- 什么配置或状态； +- 执行什么行为； +- 对外可观察的核心结果。 + +不得把实现函数名或已有测试脚本退出零当作产品目标。 + + +## 8. 前置条件 + +前置条件必须可验证并区分于测试步骤。每项应说明所需状态，而不是隐含测试结果。公共环境条件写在顶层说明书，用例只写本用例特有条件。 + +若前置条件失败，执行器记录命令证据并把用例标为 `BLOCKED`；不能把 setup错误记为产品 `FAIL`。 + + +## 9. 测试数据 + +测试数据优先使用 JSON代码块，固定协议字段、边界值和预期值。随机数据必须说明生成方式并在结果中保存实际值。安全测试地址应使用 RFC文档网段，避免依赖执行主机随机地址。 + + +## 10. 步骤和预期结果 + +每条用例推荐 3至8个逻辑步骤。一个步骤可以执行多条机械命令，但只验证一个阶段性结果。每一步必须： + +1. 有显式锚点； +2. 描述执行动作； +3. 描述可观察、可比较的预期结果； +4. 能产生至少一条命令证据。 + +不单独编写失败判据：只要实际结果没有完全满足预期，该步骤即为 `FAIL`，用例即为 `FAIL`。 + +避免使用“正常”“正确”“无异常”等模糊结果，应写为具体状态、字段、地址、摘要、计数或响应码。 + + +## 11. 后置条件 + +后置条件描述： + +- 清除哪些测试数据； +- 停止或保留哪些服务； +- 如何恢复修改的 policy和配置； +- 哪些状态供后续用例复用； +- 清理失败时如何记录。 + + +## 12. Proxy Protocol样例 + +````markdown + +# TC-GW-PP-001：启用PP的应用端口接收正确客户端地址 + + +## 测试目标 + +验证Gateway启用Inbound Proxy Protocol且应用端口声明`pp=true`时， +上游Proxy v1客户端地址被转发给应用，后续HTTP请求正常完成。 + + +## 前置条件 + +1. Gateway配置`inbound_pp_enabled=true`。 +2. guest已注册，端口8443配置`pp=true`。 +3. capture backend ready，Gateway已加载该instance的port policy。 + + +## 测试数据 + +```json +{ + "source": "198.51.100.27:45678", + "destination": "203.0.113.10:443", + "app_port": 8443 +} +``` + + +## 测试步骤 + + +### Step 1：检查生效策略 + +查询guest和Gateway中的端口策略。 + +**预期结果：** Instance ID正确，端口8443存在且`pp=true`。 + + +### Step 2：清理后端记录 + +清除capture backend历史记录。 + +**预期结果：** backend ready且记录数为0。 + + +### Step 3：发送请求 + +发送Proxy v1 header，在同一连接完成TLS和HTTP请求。 + +**预期结果：** TLS成功，HTTP返回200，request ID匹配。 + + +### Step 4：检查捕获结果 + +查询capture backend记录。 + +**预期结果：** 恰好一条新记录，source和destination与测试数据一致，HTTP请求完整。 +```` + + +## 13. 评审清单 + +提交前确认： + +- 用例目录、ID和索引一致； +- 显式锚点存在且唯一； +- 每条用例引用需求或风险； +- 目标只有一个核心行为； +- 前置条件可验证； +- 每一步有精确预期； +- 步骤数量合理且能产生原始证据； +- 模拟和真实硬件要求明确； +- 后置条件可恢复环境； +- 不包含密钥、token和环境私密值。 diff --git a/docs/testing/test-report-output-spec.md b/docs/testing/test-report-output-spec.md new file mode 100644 index 000000000..92c48e5b2 --- /dev/null +++ b/docs/testing/test-report-output-spec.md @@ -0,0 +1,274 @@ + + + + +# dstack 测试报告输出规范 + +本文规定 AI执行会话、用例总结、run汇总、附件、引用、打包和单文件HTML的格式。测试方法见[dstack 测试方法](dstack-test-methodology.md#dstack-test-methodology)，用例格式见[测试用例编写规范](test-case-authoring-spec.md#dstack-test-case-authoring-spec)。 + + +## 1. 输出原则 + +1. 每条用例由一个独立Codex或Claude会话执行； +2. Agent原生JSONL会话是命令、tool call和原始输出的主要证据； +3. Agent只写一个浅层`result.json`，不在其中复制命令输出； +4. `runner.json`、run汇总、时间、退出码和摘要由工具生成； +5. 公共版本与环境只在run顶层记录； +6. 截图、日志和二进制文件作为独立附件； +7. 所有产物保留稳定锚点并可内联为单个HTML文件。 + + +## 2. 统一命令 + +工具对外统一命名为`dstack-test`： + +```bash +dstack-test run-case +dstack-test run-plan +dstack-test finalize +dstack-test validate +dstack-test package +dstack-test render +``` + +参数统一使用kebab-case。Agent通过`--agent codex`或`--agent claude`选择，不使用独立的`--codex`、`--claude`开关。 + + +## 3. 单用例执行 + +```bash +dstack-test run-case \ + --agent codex \ + --plan path/to/plan \ + --case tc-gw-pp-001 \ + --run-id run-20260723-001 \ + --workdir path/to/repository \ + -- "附加执行要求" +``` + +runner自动把顶层说明书、`case.md`、结果目录、统一状态规则和总结schema加入Agent prompt。Agent必须先读说明书，再按用例执行；不得修改Plan规格。 + + +## 4. 结果目录 + +```text +/results// +├── prompt.md +├── session.jsonl +├── agent-stderr.log +├── runner.json +├── result.json +├── artifacts/ +└── SHA256SUMS + +/results// +├── context.json # 可选 +├── run.json +└── SHA256SUMS +``` + +不同用例写不同目录，因此可以并行执行。已存在的非空run目录默认不得覆盖；显式重跑需使用`--overwrite`。 + + +## 5. `session.jsonl` + +runner直接保存Agent CLI的原生事件流： + +- Codex：`codex exec --json`； +- Claude：`claude --print --verbose --output-format stream-json`。 + +每行必须是一个完整JSON对象。runner不改写或摘要stdout中的事件。命令、tool call、tool result、Agent消息和错误均保留在会话中。Claude和Codex格式不同，由HTML渲染器的adapter在读取时统一展示，原文件保持不变。 + +Agent应在步骤开始和结束的消息中包含完整step ID，例如： + +```text +[DSTACK-TEST-STEP-START tc-gw-pp-001-step-01] +[DSTACK-TEST-STEP-END tc-gw-pp-001-step-01 PASS] +``` + +渲染器据此链接步骤与会话事件。没有标记时，可搜索step ID；仍找不到时显示`WHOLE_SESSION_FALLBACK`，并保留整个会话作为证据。 + + +## 6. `runner.json` + +`runner.json`只能由`dstack-test`生成： + +```json +{ + "schema_version": "1.0", + "run_id": "run-20260723-001", + "case_id": "tc-gw-pp-001", + "agent": { + "type": "codex", + "model": "gpt-5.4" + }, + "session": { + "format": "codex-jsonl", + "path": "session.jsonl", + "events": 137 + }, + "prompt_path": "prompt.md", + "result_path": "result.json", + "started_at": "2026-07-23T08:10:00.000Z", + "finished_at": "2026-07-23T08:17:15.000Z", + "duration_ms": 435000, + "exit_code": 0, + "result_valid": true, + "result_error": null +} +``` + +模型名优先从会话事件提取，其次使用显式`--model`，否则填写`unknown`，不得猜测。 + +runner退出码描述Agent执行基础设施，不等于测试状态。Agent正常完成且写出合法`FAIL`结果时，runner仍可成功退出。 + + +## 7. 浅层 `result.json` + +Agent在结束前原子写入： + +```json +{ + "schema_version": "1.0", + "case_id": "tc-gw-pp-001", + "status": "PASS", + "summary": "Gateway解析入站Proxy v1地址并向启用PP的应用端口正确转发。", + "steps": [ + { + "id": "tc-gw-pp-001-step-01", + "status": "PASS", + "observed": "Gateway缓存中的端口8443配置为pp=true。" + }, + { + "id": "tc-gw-pp-001-step-02", + "status": "PASS", + "observed": "后端记录已清空，初始记录数为0。" + } + ], + "artifacts": [ + { + "name": "backend-capture", + "path": "artifacts/backend-capture.json" + } + ], + "remarks": "" +} +``` + +约束： + +- 状态只能是`PASS`、`FAIL`、`BLOCKED`、`NOT_RUN`或`SKIPPED`； +- 不得使用`PARTIAL`； +- `PASS`要求至少一个步骤，且所有步骤均为`PASS`； +- step ID必须来自`case.md`； +- `observed`只写简短观察总结，不复制命令原文； +- artifact路径必须位于本次结果目录内； +- 不能嵌入token、私钥或密钥材料。 + + +## 8. 附件 + +截图、长日志、JSON响应和二进制捕获写入`artifacts/`。`result.json`只保存名称和相对路径。finalize阶段生成`SHA256SUMS`；HTML根据MIME类型： + +- 图片：内联展示并提供下载； +- 文本和JSON：默认折叠展示原文； +- 其他二进制：内联为data URI下载链接。 + +附件不能使用绝对路径或`..`逃逸结果目录。 + + +## 9. `run.json` + +全部用例结束后，`dstack-test finalize`扫描各用例结果并生成： + +```json +{ + "schema_version": "1.0", + "id": "run-20260723-001", + "anchor": "run-20260723-001", + "plan_id": "plan-gateway-proxy-protocol", + "status": "COMPLETED", + "started_at": "2026-07-23T08:10:00.000Z", + "finished_at": "2026-07-23T09:12:00.000Z", + "executors": [ + {"type": "codex", "model": "gpt-5.4"} + ], + "software_under_test": {}, + "environment": {}, + "summary": { + "total": 1, + "completed": 1, + "by_status": { + "PASS": {"count": 1, "case_refs": ["#result-tc-gw-pp-001"]}, + "FAIL": {"count": 0, "case_refs": []}, + "BLOCKED": {"count": 0, "case_refs": []}, + "NOT_RUN": {"count": 0, "case_refs": []}, + "SKIPPED": {"count": 0, "case_refs": []} + } + }, + "case_results": [ + { + "id": "tc-gw-pp-001", + "anchor": "result-tc-gw-pp-001", + "status": "PASS", + "result_path": "../../01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-20260723-001/result.json" + } + ] +} +``` + +公共软件版本和环境由`--context `提供。只有特殊旧版组件写入单用例`version_overrides`。 + + +## 10. 校验 + +```bash +dstack-test validate --plan --run-id +``` + +至少校验： + +1. Plan目录、ID、锚点和索引顺序； +2. `session.jsonl`每个非空行均为JSON对象； +3. `runner.json`和`result.json`存在且case/run ID一致； +4. 用例和步骤状态一致； +5. artifact路径合法且文件存在； +6. 顶层统计可由用例结果重新计算； +7. 完成run不存在缺失用例； +8. `SHA256SUMS`与文件内容一致。 + + +## 11. 打包 + +```bash +dstack-test package \ + --plan \ + --run-id \ + --output -.tar.gz +``` + +支持`.tar.gz`、`.tgz`、`.tar`和`.zip`。工具必须先校验，再打包完整Plan及指定run结果； +不得夹带同一Plan下其他历史run的结果。 + + +## 12. 单文件HTML + +```bash +dstack-test render \ + --plan \ + --run-id \ + --output report.html +``` + +HTML必须离线可用并内联全部CSS、JavaScript、会话事件、文本、JSON、图片和可下载附件。展示内容包括： + +- 章／节／用例三级目录； +- 测试说明书和`case.md`原始要求； +- 公共版本、环境和Agent模型； +- 状态统计、搜索和状态过滤； +- 每条用例总结及逐步observed结果； +- step到对应会话事件的跳转； +- 原始Agent消息、tool call、命令输出和错误的交互式折叠； +- 所有附件； +- 完整原始会话折叠区； +- 稳定交叉引用锚点。 diff --git a/tools/dstack-test/README.md b/tools/dstack-test/README.md new file mode 100644 index 000000000..fd2001bb4 --- /dev/null +++ b/tools/dstack-test/README.md @@ -0,0 +1,81 @@ + + + +# dstack-test + +`dstack-test` runs each indexed test case in a fresh Codex or Claude session, +preserves the agent's native JSONL event stream, validates shallow case +summaries, materializes run-level statistics, packages results, and renders one +self-contained interactive HTML report. + +## Run one case + +```bash +tools/dstack-test/dstack-test run-case \ + --agent codex \ + --plan path/to/plan \ + --case tc-gw-pp-001 \ + --run-id run-20260723-001 \ + --workdir "$PWD" \ + -- "Do not restart the physical host" +``` + +Use `--agent claude` for Claude Code. Additional CLI options may be passed with +repeatable `--agent-arg`. The runner writes only below: + +```text +/results// +├── prompt.md +├── session.jsonl +├── agent-stderr.log +├── runner.json +├── result.json # written by the agent +├── artifacts/ +└── SHA256SUMS +``` + +## Run the indexed plan + +```bash +tools/dstack-test/dstack-test run-plan \ + --agent codex \ + --plan path/to/plan \ + --run-id run-20260723-001 \ + --context run-context.json \ + -- "Follow the environment restrictions in README.md" +``` + +Every case receives an independent agent session. `run-plan` continues after an +agent/runner failure and finalizes an `INCOMPLETE` run when necessary. + +## Finalize and validate + +```bash +tools/dstack-test/dstack-test finalize \ + --plan path/to/plan --run-id run-20260723-001 \ + --context run-context.json + +tools/dstack-test/dstack-test validate \ + --plan path/to/plan --run-id run-20260723-001 +``` + +## Package and render + +```bash +tools/dstack-test/dstack-test package \ + --plan path/to/plan --run-id run-20260723-001 \ + --output report.tar.gz + +tools/dstack-test/dstack-test render \ + --plan path/to/plan --run-id run-20260723-001 \ + --output report.html +``` + +The renderer uses only the Python standard library. The generated HTML embeds +all CSS, JavaScript, raw session events, text, JSON, images, and downloadable +binary artifacts. It has a chapter/section/case table of contents, status +filters, search, cross-reference anchors, and interactive expansion of raw +session evidence. + +`package` includes the plan and the selected run only; results from other run +IDs are deliberately excluded. diff --git a/tools/dstack-test/dstack-test b/tools/dstack-test/dstack-test new file mode 100755 index 000000000..88478fef5 --- /dev/null +++ b/tools/dstack-test/dstack-test @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +exec python3 "$SCRIPT_DIR/dstack_test.py" "$@" diff --git a/tools/dstack-test/dstack_test.py b/tools/dstack-test/dstack_test.py new file mode 100755 index 000000000..87a7f6e82 --- /dev/null +++ b/tools/dstack-test/dstack_test.py @@ -0,0 +1,633 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: D100, D101, D102, D103 +"""Run, validate, package, and render dstack test plans.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +import zipfile +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import render + +TERMINAL_STATUS = ("PASS", "FAIL", "BLOCKED", "NOT_RUN", "SKIPPED") + + +class DstackTestError(Exception): + """A user-facing dstack-test error.""" + + +def utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def atomic_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", dir=path.parent, delete=False + ) as output: + json.dump(value, output, ensure_ascii=False, indent=2) + output.write("\n") + temporary = Path(output.name) + temporary.replace(path) + + +def find_case(plan: render.Plan, value: str) -> render.CaseEntry: + candidate = Path(value).resolve() + matches = [ + case for case in plan.cases if case.id == value or case.path == candidate + ] + if len(matches) != 1: + raise DstackTestError( + f"expected one indexed case for {value!r}, found {len(matches)}" + ) + return matches[0] + + +def validate_summary(case: render.CaseEntry, path: Path) -> dict[str, Any]: + result = render.load_json(path) + if result.get("case_id") != case.id: + raise DstackTestError(f"result case_id must be {case.id!r}: {path}") + status = result.get("status") + if status not in TERMINAL_STATUS: + raise DstackTestError(f"invalid result status {status!r}: {path}") + if not isinstance(result.get("summary"), str) or not result["summary"].strip(): + raise DstackTestError(f"result summary must be non-empty: {path}") + steps = result.get("steps", []) + if not isinstance(steps, list): + raise DstackTestError(f"result steps must be an array: {path}") + seen: set[str] = set() + for step in steps: + if not isinstance(step, dict): + raise DstackTestError(f"invalid step in {path}") + step_id = step.get("id") + if not isinstance(step_id, str) or not step_id.startswith(case.id + "-step-"): + raise DstackTestError(f"invalid step ID {step_id!r} in {path}") + if step_id in seen: + raise DstackTestError(f"duplicate step ID {step_id!r} in {path}") + seen.add(step_id) + if step.get("status") not in TERMINAL_STATUS: + raise DstackTestError(f"invalid status for {step_id} in {path}") + if not isinstance(step.get("observed"), str) or not step["observed"].strip(): + raise DstackTestError(f"missing observed summary for {step_id} in {path}") + if status == "PASS" and ( + not steps or any(step["status"] != "PASS" for step in steps) + ): + raise DstackTestError( + f"PASS result requires one or more all-PASS steps: {path}" + ) + artifacts = result.get("artifacts", []) + if not isinstance(artifacts, list): + raise DstackTestError(f"artifacts must be an array: {path}") + for artifact in artifacts: + if not isinstance(artifact, dict) or not artifact.get("path"): + raise DstackTestError(f"invalid artifact in {path}") + artifact_path = render.safe_path( + path.parent, artifact["path"], "result artifact" + ) + try: + artifact_path.relative_to(path.parent.resolve()) + except ValueError as error: + raise DstackTestError( + f"artifact escapes result directory: {artifact_path}" + ) from error + if not artifact_path.is_file(): + raise DstackTestError(f"missing result artifact: {artifact_path}") + return result + + +def validate_session(path: Path) -> tuple[int, str | None]: + if not path.is_file() or path.stat().st_size == 0: + raise DstackTestError(f"missing or empty agent session: {path}") + count = 0 + model: str | None = None + with path.open(encoding="utf-8") as source: + for line_no, line in enumerate(source, 1): + if not line.strip(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError as error: + raise DstackTestError( + f"invalid session JSONL at {path}:{line_no}: {error}" + ) from error + if not isinstance(event, dict): + raise DstackTestError( + f"session event is not an object at {path}:{line_no}" + ) + count += 1 + model = model or extract_model(event) + if count == 0: + raise DstackTestError(f"agent session contains no events: {path}") + return count, model + + +def extract_model(value: Any) -> str | None: + if isinstance(value, dict): + for key in ("model", "model_name"): + candidate = value.get(key) + if isinstance(candidate, str) and candidate: + return candidate + for item in value.values(): + if found := extract_model(item): + return found + elif isinstance(value, list): + for item in value: + if found := extract_model(item): + return found + return None + + +def result_schema_text(case_id: str) -> str: + return f'''{{ + "schema_version": "1.0", + "case_id": "{case_id}", + "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", + "summary": "简短总结", + "steps": [ + {{"id": "{case_id}-step-01", "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", "observed": "实际观察总结"}} + ], + "artifacts": [{{"name": "附件名称", "path": "artifacts/相对路径"}}], + "remarks": "" +}}''' + + +def build_prompt( + plan: render.Plan, case: render.CaseEntry, result_dir: Path, user_prompt: str +) -> str: + return f"""你正在执行一个dstack规范测试用例。 + +测试计划说明：{plan.guide_path} +测试用例：{case.spec_path} +执行结果目录：{result_dir} +完整会话由外部runner自动保存，你不需要手工复制命令输出。 + +执行规则： +1. 先阅读测试计划说明，再阅读case.md。 +2. 不得修改测试计划、index.json、case.md、fixtures或测试脚本。 +3. 只能把测试产物写到本次结果目录或说明书明确允许的临时目录。 +4. 严格按case.md逐步执行；每一步开始和结束时，在回复中包含对应step ID，方便会话索引。 +5. 实际命令必须通过工具执行，不得用推测或总结代替执行。 +6. 只要实际结果没有完全满足预期，该步骤和用例就是FAIL。 +7. 前置环境阻止被测行为开始时为BLOCKED；不得使用PARTIAL。 +8. 不得把模拟结果当作真实硬件结果。 +9. 不得把token、私钥、密钥材料或其他secret输出到会话或附件。 +10. 截图、日志和其他附件写入{result_dir / "artifacts"}。 +11. 结束前把总结原子写入{result_dir / "result.json"}，不得写到其他位置。 +12. result.json必须是以下浅层结构，不要嵌入命令输出： +{result_schema_text(case.id)} +13. 最终回复必须说明用例状态和result.json路径。 + +调用方附加要求： +{user_prompt.strip() or "无"} +""" + + +def agent_command( + agent: str, model: str | None, workdir: Path, prompt: str, extra: list[str] +) -> list[str]: + if agent == "codex": + command = ["codex", "exec", "--json", "--color", "never", "-C", str(workdir)] + if model: + command += ["--model", model] + command += extra + command += [prompt] + return command + if agent == "claude": + command = ["claude", "--print", "--verbose", "--output-format", "stream-json"] + if model: + command += ["--model", model] + command += extra + command += [prompt] + return command + raise DstackTestError(f"unsupported agent: {agent}") + + +def run_case( + plan: render.Plan, + case: render.CaseEntry, + run_id: str, + agent: str, + model: str | None, + workdir: Path, + prompt: str, + extra: list[str], + overwrite: bool, +) -> dict[str, Any]: + result_dir = case.path / "results" / run_id + if result_dir.exists() and any(result_dir.iterdir()) and not overwrite: + raise DstackTestError( + f"result directory is not empty; pass --overwrite to replace it: {result_dir}" + ) + if overwrite and result_dir.exists(): + shutil.rmtree(result_dir) + (result_dir / "artifacts").mkdir(parents=True, exist_ok=True) + full_prompt = build_prompt(plan, case, result_dir, prompt) + (result_dir / "prompt.md").write_text(full_prompt, encoding="utf-8") + session_path = result_dir / "session.jsonl" + stderr_path = result_dir / "agent-stderr.log" + started_at = utc_now() + start = time.monotonic_ns() + command = agent_command(agent, model, workdir, full_prompt, extra) + binary = shutil.which(command[0]) + if not binary: + raise DstackTestError(f"agent CLI not found: {command[0]}") + with session_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + completed = subprocess.run( + command, cwd=workdir, stdout=stdout, stderr=stderr, check=False + ) + duration_ms = (time.monotonic_ns() - start) // 1_000_000 + finished_at = utc_now() + event_count, detected_model = validate_session(session_path) + result_path = result_dir / "result.json" + result_valid = False + result_error = None + try: + result = validate_summary(case, result_path) + result_valid = True + except (DstackTestError, render.ReportError) as error: + result = None + result_error = str(error) + runner = { + "schema_version": "1.0", + "run_id": run_id, + "case_id": case.id, + "agent": {"type": agent, "model": detected_model or model or "unknown"}, + "session": { + "format": f"{agent}-jsonl", + "path": "session.jsonl", + "events": event_count, + }, + "prompt_path": "prompt.md", + "result_path": "result.json", + "started_at": started_at, + "finished_at": finished_at, + "duration_ms": duration_ms, + "exit_code": completed.returncode, + "result_valid": result_valid, + "result_error": result_error, + } + atomic_json(result_dir / "runner.json", runner) + write_sha256s(result_dir) + if completed.returncode != 0: + raise DstackTestError( + f"{agent} exited with {completed.returncode}; session preserved at {session_path}" + ) + if not result_valid: + raise DstackTestError( + f"agent completed without a valid result.json: {result_error}" + ) + return {"case": case.id, "status": result["status"], "result_dir": str(result_dir)} + + +def write_sha256s(root: Path) -> None: + entries: list[str] = [] + for path in sorted( + p for p in root.rglob("*") if p.is_file() and p.name != "SHA256SUMS" + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + entries.append(f"{digest} {path.relative_to(root).as_posix()}") + (root / "SHA256SUMS").write_text( + "\n".join(entries) + ("\n" if entries else ""), encoding="utf-8" + ) + + +def verify_sha256s(root: Path) -> None: + sums = root / "SHA256SUMS" + if not sums.is_file(): + raise DstackTestError(f"missing SHA256SUMS: {sums}") + expected_files: set[Path] = set() + for line_no, line in enumerate(sums.read_text(encoding="utf-8").splitlines(), 1): + match = re.fullmatch(r"([0-9a-f]{64}) (.+)", line) + if not match: + raise DstackTestError(f"invalid SHA256SUMS line at {sums}:{line_no}") + path = render.safe_path(root, match.group(2), "SHA256SUMS") + try: + path.relative_to(root.resolve()) + except ValueError as error: + raise DstackTestError( + f"SHA256SUMS path escapes result directory: {path}" + ) from error + if not path.is_file(): + raise DstackTestError(f"SHA256SUMS references missing file: {path}") + if hashlib.sha256(path.read_bytes()).hexdigest() != match.group(1): + raise DstackTestError(f"SHA-256 mismatch: {path}") + expected_files.add(path) + actual_files = { + path.resolve() + for path in root.rglob("*") + if path.is_file() and path.name != "SHA256SUMS" + } + if expected_files != actual_files: + missing = sorted( + str(path.relative_to(root)) for path in actual_files - expected_files + ) + raise DstackTestError(f"files missing from SHA256SUMS in {root}: {missing}") + + +def context_value(path: Path | None) -> dict[str, Any]: + if path is None: + return {} + value = render.load_json(path) + return value + + +def finalize_run( + plan: render.Plan, run_id: str, context: dict[str, Any] | None = None +) -> dict[str, Any]: + case_results: list[dict[str, Any]] = [] + by_status = {status: {"count": 0, "case_refs": []} for status in TERMINAL_STATUS} + started: list[str] = [] + finished: list[str] = [] + executors: list[dict[str, Any]] = [] + incomplete = False + for case in plan.cases: + result_dir = case.path / "results" / run_id + result_path = result_dir / "result.json" + runner_path = result_dir / "runner.json" + if not result_path.is_file() or not runner_path.is_file(): + incomplete = True + continue + result = validate_summary(case, result_path) + runner = render.load_json(runner_path) + validate_session(result_dir / "session.jsonl") + status = result["status"] + by_status[status]["count"] += 1 + by_status[status]["case_refs"].append(f"#result-{case.id}") + case_results.append( + { + "id": case.id, + "anchor": f"result-{case.id}", + "status": status, + "result_path": os.path.relpath( + result_path, plan.root / "results" / run_id + ), + } + ) + if runner.get("started_at"): + started.append(runner["started_at"]) + if runner.get("finished_at"): + finished.append(runner["finished_at"]) + if runner.get("agent") and runner["agent"] not in executors: + executors.append(runner["agent"]) + run_dir = plan.root / "results" / run_id + total = len(plan.cases) + run = { + "schema_version": "1.0", + "id": run_id, + "anchor": run_id, + "plan_id": plan.index["id"], + "status": "INCOMPLETE" + if incomplete or len(case_results) != total + else "COMPLETED", + "started_at": min(started) if started else None, + "finished_at": max(finished) if finished else None, + "executors": executors, + "software_under_test": (context or {}).get("software_under_test", {}), + "environment": (context or {}).get("environment", {}), + "summary": { + "total": total, + "completed": len(case_results), + "by_status": by_status, + }, + "case_results": case_results, + } + atomic_json(run_dir / "run.json", run) + if context: + atomic_json(run_dir / "context.json", context) + write_sha256s(run_dir) + return run + + +def validate_run(plan: render.Plan, run_id: str) -> dict[str, Any]: + run_path = plan.root / "results" / run_id / "run.json" + run = render.load_json(run_path) + if run.get("plan_id") != plan.index["id"] or run.get("id") != run_id: + raise DstackTestError("run/plan ID mismatch") + actual = {status: [] for status in TERMINAL_STATUS} + for item in run.get("case_results", []): + case = find_case(plan, str(item.get("id"))) + result_path = (run_path.parent / str(item.get("result_path", ""))).resolve() + try: + result_path.relative_to(plan.root) + except ValueError as error: + raise DstackTestError( + f"case result escapes plan root: {result_path}" + ) from error + result = validate_summary(case, result_path) + result_dir = result_path.parent + validate_session(result_dir / "session.jsonl") + runner = render.load_json(result_dir / "runner.json") + if not runner.get("result_valid"): + raise DstackTestError(f"runner marks invalid result for {case.id}") + verify_sha256s(result_dir) + actual[result["status"]].append(f"#result-{case.id}") + summary = run.get("summary", {}) + if summary.get("total") != len(plan.cases): + raise DstackTestError("summary total does not match plan") + for status in TERMINAL_STATUS: + item = summary.get("by_status", {}).get(status, {}) + if ( + item.get("count") != len(actual[status]) + or item.get("case_refs") != actual[status] + ): + raise DstackTestError(f"summary mismatch for {status}") + verify_sha256s(run_path.parent) + return { + "status": "valid", + "plan_id": plan.index["id"], + "run_id": run_id, + "cases": len(run.get("case_results", [])), + } + + +def package_plan(plan: render.Plan, run_id: str, output: Path) -> None: + validate_run(plan, run_id) + output.parent.mkdir(parents=True, exist_ok=True) + files = [] + for path in sorted(item for item in plan.root.rglob("*") if item.is_file()): + relative = path.relative_to(plan.root) + parts = relative.parts + # A package is one immutable execution report, not an archive of all + # historical executions that happen to live beside the plan. + if "results" in parts: + index = parts.index("results") + if len(parts) > index + 1 and parts[index + 1] != run_id: + continue + files.append((path, Path("plan") / relative)) + suffix = output.name.lower() + if suffix.endswith(".tar.gz") or suffix.endswith(".tgz"): + with tarfile.open(output, "w:gz") as archive: + for path, archive_path in files: + archive.add(path, arcname=archive_path, recursive=False) + elif suffix.endswith(".tar"): + with tarfile.open(output, "w") as archive: + for path, archive_path in files: + archive.add(path, arcname=archive_path, recursive=False) + elif suffix.endswith(".zip"): + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path, archive_path in files: + archive.write(path, archive_path) + else: + raise DstackTestError("package output must end in .tar.gz, .tgz, .tar, or .zip") + + +def add_common_plan(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--plan", type=Path, required=True, help="test-plan directory") + parser.add_argument("--run-id", required=True, help="shared run identifier") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="dstack-test", description=__doc__) + sub = parser.add_subparsers(dest="subcommand", required=True) + run_case_parser = sub.add_parser( + "run-case", help="run one indexed case in a fresh AI session" + ) + add_common_plan(run_case_parser) + run_case_parser.add_argument( + "--case", required=True, help="case ID or indexed case directory" + ) + run_case_parser.add_argument("--agent", choices=("codex", "claude"), required=True) + run_case_parser.add_argument("--model") + run_case_parser.add_argument("--workdir", type=Path, default=Path.cwd()) + run_case_parser.add_argument("--prompt-file", type=Path) + run_case_parser.add_argument("--agent-arg", action="append", default=[]) + run_case_parser.add_argument("--overwrite", action="store_true") + run_case_parser.add_argument("prompt", nargs=argparse.REMAINDER) + + run_plan = sub.add_parser( + "run-plan", help="run every indexed case in its own AI session" + ) + add_common_plan(run_plan) + run_plan.add_argument("--agent", choices=("codex", "claude"), required=True) + run_plan.add_argument("--model") + run_plan.add_argument("--workdir", type=Path, default=Path.cwd()) + run_plan.add_argument("--prompt-file", type=Path) + run_plan.add_argument( + "--context", + type=Path, + help="run-level JSON with software_under_test/environment", + ) + run_plan.add_argument("--agent-arg", action="append", default=[]) + run_plan.add_argument("--overwrite", action="store_true") + run_plan.add_argument("prompt", nargs=argparse.REMAINDER) + + finalize = sub.add_parser("finalize", help="materialize run.json from case results") + add_common_plan(finalize) + finalize.add_argument("--context", type=Path) + + validate = sub.add_parser("validate", help="validate a finalized test run") + add_common_plan(validate) + + package = sub.add_parser( + "package", help="validate and package a test plan and its results" + ) + add_common_plan(package) + package.add_argument("--output", type=Path, required=True) + + render_parser = sub.add_parser( + "render", help="render a validated run to one offline HTML file" + ) + add_common_plan(render_parser) + render_parser.add_argument("--output", type=Path, required=True) + return parser + + +def prompt_text(args: argparse.Namespace) -> str: + values = list(args.prompt) + if values and values[0] == "--": + values = values[1:] + text = " ".join(values) + if args.prompt_file: + file_text = args.prompt_file.read_text(encoding="utf-8") + text = f"{file_text}\n\n{text}" if text else file_text + return text + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + plan = render.load_plan(args.plan) + if args.subcommand == "run-case": + value = run_case( + plan, + find_case(plan, args.case), + args.run_id, + args.agent, + args.model, + args.workdir.resolve(), + prompt_text(args), + args.agent_arg, + args.overwrite, + ) + elif args.subcommand == "run-plan": + values = [] + for case in plan.cases: + try: + values.append( + run_case( + plan, + case, + args.run_id, + args.agent, + args.model, + args.workdir.resolve(), + prompt_text(args), + args.agent_arg, + args.overwrite, + ) + ) + except DstackTestError as error: + values.append({"case": case.id, "runner_error": str(error)}) + run = finalize_run(plan, args.run_id, context_value(args.context)) + value = {"cases": values, "run": run} + elif args.subcommand == "finalize": + value = finalize_run(plan, args.run_id, context_value(args.context)) + elif args.subcommand == "validate": + value = validate_run(plan, args.run_id) + elif args.subcommand == "package": + package_plan(plan, args.run_id, args.output) + value = { + "status": "packaged", + "output": str(args.output), + "sha256": hashlib.sha256(args.output.read_bytes()).hexdigest(), + } + elif args.subcommand == "render": + validate_run(plan, args.run_id) + run, results = render.load_session_results(plan, args.run_id) + rendered = render.render_report(plan, run, results) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + value = { + "status": "rendered", + "output": str(args.output), + "bytes": args.output.stat().st_size, + } + else: + raise AssertionError(args.subcommand) + print(json.dumps(value, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (DstackTestError, render.ReportError) as error: + print( + json.dumps({"status": "error", "message": str(error)}, ensure_ascii=False), + file=sys.stderr, + ) + raise SystemExit(2) diff --git a/tools/dstack-test/render.py b/tools/dstack-test/render.py new file mode 100755 index 000000000..739ef4bba --- /dev/null +++ b/tools/dstack-test/render.py @@ -0,0 +1,932 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: D100, D101, D102, D103 +"""Validate and render a dstack test-plan run as one self-contained HTML file.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import html +import json +import mimetypes +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +STATUS = ("PASS", "FAIL", "BLOCKED", "NOT_RUN", "SKIPPED") +ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +ANCHOR_LINE_RE = re.compile(r'^\s*\s*$') +HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$") +LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") +CODE_RE = re.compile(r"`([^`]+)`") +BOLD_RE = re.compile(r"\*\*([^*]+)\*\*") + + +class ReportError(Exception): + """Raised when a plan or result violates the report contract.""" + + +@dataclass +class CaseEntry: + chapter_id: str + chapter_title: str + section_id: str + section_title: str + id: str + title: str + path: Path + spec_path: Path + spec_anchor: str + priority: str = "" + requirements: list[str] = field(default_factory=list) + risks: list[str] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + + +@dataclass +class Plan: + root: Path + index: dict[str, Any] + guide_path: Path + cases: list[CaseEntry] + chapter_docs: dict[str, Path] + section_docs: dict[str, Path] + anchors: set[str] + + +def load_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as error: + raise ReportError(f"missing JSON file: {path}") from error + except json.JSONDecodeError as error: + raise ReportError(f"invalid JSON in {path}: {error}") from error + if not isinstance(value, dict): + raise ReportError(f"expected JSON object in {path}") + return value + + +def require(obj: dict[str, Any], key: str, context: str) -> Any: + if key not in obj: + raise ReportError(f"missing {key!r} in {context}") + return obj[key] + + +def check_id(value: Any, context: str) -> str: + if not isinstance(value, str) or not ID_RE.fullmatch(value): + raise ReportError(f"invalid ID {value!r} in {context}") + return value + + +def safe_path(root: Path, relative: str, context: str) -> Path: + if not isinstance(relative, str) or not relative or Path(relative).is_absolute(): + raise ReportError(f"invalid relative path {relative!r} in {context}") + candidate = (root / relative).resolve() + try: + candidate.relative_to(root.resolve()) + except ValueError as error: + raise ReportError(f"path escapes plan root in {context}: {relative}") from error + return candidate + + +def markdown_anchors(path: Path) -> set[str]: + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as error: + raise ReportError(f"missing Markdown document: {path}") from error + return { + match.group(1) + for line in text.splitlines() + if (match := ANCHOR_LINE_RE.match(line)) + } + + +def sorted_by_order(items: Any, context: str) -> list[dict[str, Any]]: + if not isinstance(items, list): + raise ReportError(f"expected array in {context}") + result: list[dict[str, Any]] = [] + orders: list[int] = [] + for item in items: + if not isinstance(item, dict): + raise ReportError(f"expected object in {context}") + order = require(item, "order", context) + if not isinstance(order, int) or order < 1: + raise ReportError(f"invalid order in {context}: {order!r}") + result.append(item) + orders.append(order) + if orders != sorted(orders) or len(orders) != len(set(orders)): + raise ReportError(f"items in {context} must have unique ascending order") + return result + + +def load_plan(root: Path) -> Plan: + root = root.resolve() + index = load_json(root / "index.json") + plan_id = check_id(require(index, "id", "index.json"), "index.json") + guide = require(index, "guide", "index.json") + if not isinstance(guide, dict): + raise ReportError("guide must be an object") + guide_path = safe_path(root, require(guide, "path", "guide"), "guide") + guide_anchor = check_id(require(guide, "anchor", "guide"), "guide") + anchors: set[str] = {plan_id} + guide_anchors = markdown_anchors(guide_path) + if guide_anchor not in guide_anchors: + raise ReportError(f"guide anchor #{guide_anchor} not found in {guide_path}") + anchors.update(guide_anchors) + + cases: list[CaseEntry] = [] + chapter_docs: dict[str, Path] = {} + section_docs: dict[str, Path] = {} + ids = {plan_id} + for chapter in sorted_by_order( + require(index, "chapters", "index.json"), "chapters" + ): + chapter_id = check_id(require(chapter, "id", "chapter"), "chapter") + if chapter_id in ids: + raise ReportError(f"duplicate ID: {chapter_id}") + ids.add(chapter_id) + chapter_title = str(require(chapter, "title", chapter_id)) + chapter_root = safe_path(root, require(chapter, "path", chapter_id), chapter_id) + document = chapter.get("document") + if isinstance(document, dict): + doc = safe_path(root, require(document, "path", chapter_id), chapter_id) + anchor = check_id(require(document, "anchor", chapter_id), chapter_id) + doc_anchors = markdown_anchors(doc) + if anchor not in doc_anchors: + raise ReportError(f"chapter anchor #{anchor} not found in {doc}") + anchors.update(doc_anchors) + chapter_docs[chapter_id] = doc + elif (chapter_root / "README.md").is_file(): + chapter_docs[chapter_id] = chapter_root / "README.md" + anchors.update(markdown_anchors(chapter_root / "README.md")) + + for section in sorted_by_order( + require(chapter, "sections", chapter_id), f"{chapter_id}.sections" + ): + section_id = check_id(require(section, "id", "section"), "section") + if section_id in ids: + raise ReportError(f"duplicate ID: {section_id}") + ids.add(section_id) + section_title = str(require(section, "title", section_id)) + section_root = safe_path( + root, require(section, "path", section_id), section_id + ) + document = section.get("document") + if isinstance(document, dict): + doc = safe_path(root, require(document, "path", section_id), section_id) + anchor = check_id(require(document, "anchor", section_id), section_id) + doc_anchors = markdown_anchors(doc) + if anchor not in doc_anchors: + raise ReportError(f"section anchor #{anchor} not found in {doc}") + anchors.update(doc_anchors) + section_docs[section_id] = doc + elif (section_root / "README.md").is_file(): + section_docs[section_id] = section_root / "README.md" + anchors.update(markdown_anchors(section_root / "README.md")) + + for case in sorted_by_order( + require(section, "cases", section_id), f"{section_id}.cases" + ): + case_id = check_id(require(case, "id", "case"), "case") + if case_id in ids: + raise ReportError(f"duplicate ID: {case_id}") + ids.add(case_id) + case_path = safe_path(root, require(case, "path", case_id), case_id) + spec = require(case, "spec", case_id) + if not isinstance(spec, dict): + raise ReportError(f"spec must be object in {case_id}") + spec_path = safe_path(root, require(spec, "path", case_id), case_id) + spec_anchor = check_id(require(spec, "anchor", case_id), case_id) + case_anchors = markdown_anchors(spec_path) + if spec_anchor not in case_anchors: + raise ReportError( + f"spec anchor #{spec_anchor} not found in {spec_path}" + ) + duplicate_anchors = anchors.intersection(case_anchors) + if duplicate_anchors: + raise ReportError( + f"duplicate Markdown anchor(s): {sorted(duplicate_anchors)}" + ) + anchors.update(case_anchors) + cases.append( + CaseEntry( + chapter_id=chapter_id, + chapter_title=chapter_title, + section_id=section_id, + section_title=section_title, + id=case_id, + title=str(require(case, "title", case_id)), + path=case_path, + spec_path=spec_path, + spec_anchor=spec_anchor, + priority=str(case.get("priority", "")), + requirements=list(case.get("requirements", [])), + risks=list(case.get("risks", [])), + tags=list(case.get("tags", [])), + ) + ) + return Plan(root, index, guide_path, cases, chapter_docs, section_docs, anchors) + + +def inline_markup(text: str) -> str: + escaped = html.escape(text, quote=False) + escaped = CODE_RE.sub(lambda m: f"{m.group(1)}", escaped) + escaped = BOLD_RE.sub(lambda m: f"{m.group(1)}", escaped) + + def link(match: re.Match[str]) -> str: + label, target = match.group(1), html.unescape(match.group(2)) + if "#" in target: + target = "#" + target.rsplit("#", 1)[1] + return f'{label}' + + return LINK_RE.sub(link, escaped) + + +def markdown_to_html(text: str) -> str: + """Render the deliberately small Markdown subset used by test plans.""" + lines = text.splitlines() + out: list[str] = [] + paragraph: list[str] = [] + list_type: str | None = None + in_code = False + code_lang = "" + code_lines: list[str] = [] + table_lines: list[str] = [] + + def flush_paragraph() -> None: + nonlocal paragraph + if paragraph: + out.append( + f"

{inline_markup(' '.join(x.strip() for x in paragraph))}

" + ) + paragraph = [] + + def close_list() -> None: + nonlocal list_type + if list_type: + out.append(f"") + list_type = None + + def flush_table() -> None: + nonlocal table_lines + if not table_lines: + return + rows = [ + [cell.strip() for cell in line.strip().strip("|").split("|")] + for line in table_lines + ] + if len(rows) >= 2 and all( + re.fullmatch(r":?-{3,}:?", c.replace(" ", "")) for c in rows[1] + ): + out.append( + '
' + + "".join(f"" for c in rows[0]) + + "" + ) + for row in rows[2:]: + out.append( + "" + + "".join(f"" for c in row) + + "" + ) + out.append("
{inline_markup(c)}
{inline_markup(c)}
") + else: + out.extend(f"

{inline_markup(line)}

" for line in table_lines) + table_lines = [] + + for line in lines + [""]: + if in_code: + if line.startswith("```"): + out.append( + f'
{html.escape(chr(10).join(code_lines))}
' + ) + in_code = False + code_lines = [] + else: + code_lines.append(line) + continue + if line.startswith("```"): + flush_paragraph() + close_list() + flush_table() + in_code = True + code_lang = line[3:].strip() + continue + if match := ANCHOR_LINE_RE.match(line): + flush_paragraph() + close_list() + flush_table() + out.append(f'') + continue + if match := HEADING_RE.match(line): + flush_paragraph() + close_list() + flush_table() + level = len(match.group(1)) + out.append(f"{inline_markup(match.group(2))}") + continue + if line.startswith("|") and line.rstrip().endswith("|"): + flush_paragraph() + close_list() + table_lines.append(line) + continue + flush_table() + if match := re.match(r"^\s*[-*]\s+(.+)$", line): + flush_paragraph() + if list_type != "ul": + close_list() + out.append("
    ") + list_type = "ul" + out.append(f"
  • {inline_markup(match.group(1))}
    • ") + continue + if match := re.match(r"^\s*\d+[.)]\s+(.+)$", line): + flush_paragraph() + if list_type != "ol": + close_list() + out.append("
      ") + list_type = "ol" + out.append(f"
    1. {inline_markup(match.group(1))}
      2. ") + continue + if not line.strip(): + flush_paragraph() + close_list() + else: + close_list() + paragraph.append(line) + return "\n".join(out) + + +def status_badge(status: str) -> str: + cls = status.lower().replace("_", "-") + return f'{html.escape(status)}' + + +def json_block(value: Any) -> str: + return f'
      {html.escape(json.dumps(value, ensure_ascii=False, indent=2))}
      ' + + +def ref_link(ref: str, label: str | None = None) -> str: + fragment = ref.rsplit("#", 1)[-1] if "#" in ref else "" + href = f"#{html.escape(fragment, quote=True)}" if fragment else "#" + return f'{html.escape(label or ref)}' + + +def resolve_ref_file(base: Path, ref: str, plan_root: Path) -> tuple[Path, str]: + path_part, _, anchor = ref.partition("#") + path = base if not path_part else (base.parent / path_part).resolve() + try: + path.relative_to(plan_root) + except ValueError as error: + raise ReportError(f"reference escapes plan root: {ref}") from error + if not path.is_file(): + raise ReportError(f"referenced file does not exist: {ref} from {base}") + return path, anchor + + +def load_evidence( + result_path: Path, ref: str, plan_root: Path +) -> tuple[dict[str, Any], Path]: + path, anchor = resolve_ref_file(result_path, ref, plan_root) + value = load_json(path) + if anchor and anchor not in (value.get("anchor"), value.get("id")): + raise ReportError(f"evidence anchor #{anchor} not found in {path}") + return value, path + + +def verify_attachment(result_path: Path, item: dict[str, Any], plan_root: Path) -> Path: + path = safe_path( + result_path.parent, require(item, "path", "attachment"), "attachment" + ) + try: + path.relative_to(plan_root) + except ValueError as error: + raise ReportError(f"attachment escapes plan root: {path}") from error + if not path.is_file(): + raise ReportError(f"missing attachment: {path}") + data = path.read_bytes() + expected_size = item.get("size_bytes") + if expected_size is not None and expected_size != len(data): + raise ReportError(f"attachment size mismatch: {path}") + expected_hash = item.get("sha256") + if expected_hash and hashlib.sha256(data).hexdigest() != expected_hash: + raise ReportError(f"attachment SHA-256 mismatch: {path}") + return path + + +def render_command(command: dict[str, Any], anchor_override: str | None = None) -> str: + anchor = anchor_override or str( + command.get("anchor") or command.get("id") or "command" + ) + title = str(command.get("command", "command")) + env = command.get("environment", {}) + meta = { + "cwd": command.get("cwd"), + "environment": env, + "started_at": command.get("started_at"), + "finished_at": command.get("finished_at"), + "duration_ms": command.get("duration_ms"), + "exit_code": command.get("exit_code"), + } + parts = [ + f'', + '
      › ', + f"{html.escape(title)}", + f' exit {html.escape(str(command.get("exit_code", "?")))}', + ] + parts.append('
      Execution metadata
      ') + parts.append(json_block(meta)) + for stream in ("stdout", "stderr"): + value = command.get(stream) + if value is not None: + parts.append( + f'
      {stream}
      {html.escape(str(value))}
      ' + ) + attachment = command.get(f"{stream}_attachment") + if attachment: + parts.append( + f"

      {stream} attachment: {ref_link(str(attachment.get('ref', '')))}

      " + ) + parts.append("
      ") + return "".join(parts) + + +def render_attachment(item: dict[str, Any], path: Path) -> str: + data = path.read_bytes() + media = str( + item.get("media_type") + or mimetypes.guess_type(path.name)[0] + or "application/octet-stream" + ) + anchor = str(item.get("anchor") or item.get("id") or f"attachment-{path.name}") + title = str(item.get("title") or path.name) + digest = hashlib.sha256(data).hexdigest() + header = f'
      {html.escape(title)}

      {html.escape(media)} · {len(data)} bytes · SHA-256 {digest}

      ' + if media.startswith("image/"): + uri = f"data:{media};base64,{base64.b64encode(data).decode()}" + return ( + header + + f'{html.escape(title)}' + ) + if media.startswith("text/") or media in ("application/json", "application/xml"): + text = data.decode("utf-8", errors="replace") + lang = "json" if media == "application/json" else "text" + return ( + header + + f'
      › View {html.escape(path.name)}
      {html.escape(text)}
      ' + ) + uri = f"data:{media};base64,{base64.b64encode(data).decode()}" + return ( + header + + f'Download inline attachment' + ) + + +def validate_case_result(case: CaseEntry, result: dict[str, Any], path: Path) -> None: + if result.get("id") != case.id: + raise ReportError(f"case ID mismatch in {path}: expected {case.id}") + status = result.get("status") + if status not in STATUS: + raise ReportError(f"invalid case status {status!r} in {path}") + steps = result.get("steps", []) + if not isinstance(steps, list): + raise ReportError(f"steps must be array in {path}") + for step in steps: + if not isinstance(step, dict) or step.get("status") not in STATUS: + raise ReportError(f"invalid step in {path}") + refs = step.get("evidence_refs", []) + commands = step.get("commands", []) + if status not in ("NOT_RUN", "SKIPPED") and not refs and not commands: + raise ReportError( + f"step {step.get('id')} has no command evidence in {path}" + ) + if status == "PASS" and any(step.get("status") != "PASS" for step in steps): + raise ReportError(f"PASS case contains non-PASS step in {path}") + if status == "PASS" and not steps: + raise ReportError(f"PASS case has no steps in {path}") + + +def session_events(path: Path) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as source: + for line_no, line in enumerate(source, 1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as error: + raise ReportError( + f"invalid session JSONL at {path}:{line_no}: {error}" + ) from error + if not isinstance(value, dict): + raise ReportError(f"session event is not an object at {path}:{line_no}") + events.append(value) + if not events: + raise ReportError(f"empty session JSONL: {path}") + return events + + +def event_title(event: dict[str, Any], number: int) -> str: + kind = str( + event.get("type") or event.get("event") or event.get("subtype") or "event" + ) + item = event.get("item") + if isinstance(item, dict) and item.get("type"): + kind += f" · {item['type']}" + message = event.get("message") + if isinstance(message, dict): + content = message.get("content") + if isinstance(content, str) and content.strip(): + kind += f" · {content.strip()[:80]}" + return f"{number:04d} · {kind}" + + +def render_session(case_id: str, events: list[dict[str, Any]]) -> str: + parts = [ + '
      › Complete agent session (' + + str(len(events)) + + ' events)
      ' + ] + for number, event in enumerate(events, 1): + anchor = f"session-{case_id}-event-{number}" + parts.append( + f'
      › {html.escape(event_title(event, number))}{json_block(event)}
      ' + ) + parts.append("
      ") + return "".join(parts) + + +def step_session_ref(step_id: str, events: list[dict[str, Any]], case_id: str) -> str: + for number, event in enumerate(events, 1): + if step_id in json.dumps(event, ensure_ascii=False): + return f'Jump to session evidence · event {number}' + return 'Evidence mapping: whole session fallback' + + +def render_case_result( + case: CaseEntry, result: dict[str, Any], result_path: Path, plan: Plan +) -> str: + status = str(result["status"]) + result_dir = result_path.parent + runner_path = result_dir / "runner.json" + session_path = result_dir / "session.jsonl" + if not runner_path.is_file(): + raise ReportError(f"missing runner.json for {case.id}") + runner = load_json(runner_path) + events = session_events(session_path) + parts = [ + f'
      ' + ] + parts.append( + f'

      Execution result

      {status_badge(status)}
      ' + ) + parts.append( + f'
      {html.escape(str(result.get("summary", "")))}
      ' + ) + parts.append( + '
      › Runner metadata' + + json_block(runner) + + "
      " + ) + if result.get("version_overrides"): + parts.append( + '
      › Version overrides' + + json_block(result["version_overrides"]) + + "
      " + ) + for step in result.get("steps", []): + step_id = str(step.get("id", "step")) + step_status = str(step.get("status", "")) + parts.append( + f'

      {html.escape(step_id)}

      {status_badge(step_status)}
      ' + ) + parts.append( + f'
      Observed

      {html.escape(str(step.get("observed", "")))}

      ' + ) + parts.append(step_session_ref(step_id, events, case.id)) + parts.append("
      ") + artifacts = result.get("artifacts", []) + if artifacts: + parts.append( + '
      › Artifacts' + ) + for number, artifact in enumerate(artifacts, 1): + if not isinstance(artifact, dict) or not artifact.get("path"): + raise ReportError(f"invalid artifact in {result_path}") + path = safe_path(result_dir, str(artifact["path"]), "result artifact") + try: + path.relative_to(result_dir.resolve()) + except ValueError as error: + raise ReportError( + f"artifact escapes result directory: {path}" + ) from error + if not path.is_file(): + raise ReportError(f"missing artifact: {path}") + item = { + "id": f"attachment-{case.id}-{number}", + "anchor": f"attachment-{case.id}-{number}", + "title": artifact.get("name") or path.name, + "media_type": artifact.get("media_type") + or mimetypes.guess_type(path.name)[0], + } + parts.append(render_attachment(item, path)) + parts.append("
      ") + parts.append(render_session(case.id, events)) + if result.get("remarks"): + parts.append( + f'
      Remarks: {html.escape(str(result["remarks"]))}
      ' + ) + parts.append("
      ") + return "".join(parts) + + +def load_session_results( + plan: Plan, run_id: str +) -> tuple[dict[str, Any], dict[str, tuple[dict[str, Any], Path]]]: + run_path = plan.root / "results" / run_id / "run.json" + run = load_json(run_path) + if run.get("id") != run_id or run.get("plan_id") != plan.index.get("id"): + raise ReportError(f"run/plan ID mismatch in {run_path}") + listed = run.get("case_results", []) + if not isinstance(listed, list): + raise ReportError("case_results must be an array") + path_by_id: dict[str, Path] = {} + for item in listed: + if not isinstance(item, dict): + raise ReportError("case_results entries must be objects") + case_id = check_id(require(item, "id", "case_results"), "case_results") + candidate = ( + run_path.parent / str(require(item, "result_path", case_id)) + ).resolve() + try: + candidate.relative_to(plan.root) + except ValueError as error: + raise ReportError(f"case result escapes plan root: {candidate}") from error + path_by_id[case_id] = candidate + results: dict[str, tuple[dict[str, Any], Path]] = {} + for case in plan.cases: + path = path_by_id.get(case.id) + if path is None or not path.is_file(): + raise ReportError(f"missing finalized result for {case.id}") + result = load_json(path) + if result.get("case_id") != case.id or result.get("status") not in STATUS: + raise ReportError(f"invalid shallow result for {case.id}: {path}") + session_events(path.parent / "session.jsonl") + if not (path.parent / "runner.json").is_file(): + raise ReportError(f"missing runner.json for {case.id}") + results[case.id] = (result, path) + return run, results + + +def load_results( + plan: Plan, run_id: str +) -> tuple[dict[str, Any], dict[str, tuple[dict[str, Any], Path]]]: + run_path = plan.root / "results" / run_id / "run.json" + run = load_json(run_path) + if run.get("id") != run_id: + raise ReportError(f"run ID mismatch in {run_path}") + if run.get("plan_id") != plan.index.get("id"): + raise ReportError(f"plan ID mismatch in {run_path}") + listed = run.get("case_results", []) + path_by_id: dict[str, Path] = {} + if not isinstance(listed, list): + raise ReportError("case_results must be an array") + for item in listed: + if not isinstance(item, dict): + raise ReportError("case_results entries must be objects") + case_id = check_id(require(item, "id", "case_results"), "case_results") + candidate = ( + run_path.parent / str(require(item, "result_path", case_id)) + ).resolve() + try: + candidate.relative_to(plan.root) + except ValueError as error: + raise ReportError(f"case result escapes plan root: {candidate}") from error + path_by_id[case_id] = candidate + results: dict[str, tuple[dict[str, Any], Path]] = {} + for case in plan.cases: + path = path_by_id.get(case.id, case.path / "results" / run_id / "result.json") + if not path.is_file(): + raise ReportError(f"missing result for {case.id}: {path}") + result = load_json(path) + validate_case_result(case, result, path) + results[case.id] = (result, path) + actual: dict[str, list[str]] = {status: [] for status in STATUS} + for case in plan.cases: + actual[str(results[case.id][0]["status"])].append(f"#result-{case.id}") + summary = require(run, "summary", str(run_path)) + if not isinstance(summary, dict) or summary.get("total") != len(plan.cases): + raise ReportError("run summary total does not match plan") + by_status = require(summary, "by_status", "summary") + for status in STATUS: + item = by_status.get(status, {}) + if ( + item.get("count") != len(actual[status]) + or item.get("case_refs") != actual[status] + ): + raise ReportError(f"summary mismatch for {status}") + return run, results + + +def nav_html(plan: Plan, results: dict[str, tuple[dict[str, Any], Path]]) -> str: + guide_anchor = html.escape(str(plan.index["guide"]["anchor"]), quote=True) + parts = [ + f'") + return "".join(parts) + + +CSS = r""" +:root{--bg:#f5f7fb;--panel:#fff;--ink:#172033;--muted:#667085;--line:#e3e8ef;--brand:#5b5bd6;--brand2:#7c3aed;--pass:#067647;--pass-bg:#ecfdf3;--fail:#b42318;--fail-bg:#fef3f2;--blocked:#b54708;--blocked-bg:#fffaeb;--not:#475467;--not-bg:#f2f4f7;--shadow:0 10px 30px rgba(16,24,40,.08)} +*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.65 Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif}.layout{display:grid;grid-template-columns:300px minmax(0,1fr);min-height:100vh}.sidebar{position:sticky;top:0;height:100vh;overflow:auto;background:#111827;color:#e5e7eb;padding:22px 16px}.brand{font-size:19px;font-weight:750;margin-bottom:4px}.run-label{font-size:12px;color:#9ca3af;margin-bottom:18px}.search{width:100%;border:1px solid #374151;background:#1f2937;color:#fff;border-radius:9px;padding:9px 11px;margin-bottom:12px}.filter-row{display:flex;gap:5px;flex-wrap:wrap;margin-bottom:14px}.filter{border:1px solid #374151;background:#1f2937;color:#d1d5db;border-radius:999px;padding:4px 8px;font-size:11px;cursor:pointer}.filter.active{background:var(--brand);border-color:var(--brand);color:#fff}.toc a{color:#d1d5db;text-decoration:none;display:flex;align-items:center;justify-content:space-between;padding:5px 7px;border-radius:6px}.toc a:hover{background:#1f2937;color:#fff}.toc-title{text-transform:uppercase;color:#9ca3af;font-size:11px;letter-spacing:.1em;margin:10px 7px}.toc-chapter{margin-top:8px}.toc-chapter>a{font-weight:700}.toc-section{margin-left:9px}.toc-section>a{font-size:13px;font-weight:600}.toc-case{margin-left:12px;font-size:12px;gap:6px}.toc .status{font-size:9px;padding:1px 5px}.main{min-width:0;padding:36px 46px 80px;max-width:1440px;width:100%;margin:auto}.hero{background:linear-gradient(125deg,#312e81,#6d28d9 60%,#2563eb);color:#fff;padding:35px;border-radius:18px;box-shadow:var(--shadow);margin-bottom:24px}.hero h1{margin:0 0 6px;font-size:30px}.hero p{color:#ddd6fe;margin:0}.panel,.chapter,.section,.case-card{background:var(--panel);border:1px solid var(--line);border-radius:14px;box-shadow:0 3px 14px rgba(16,24,40,.04);padding:24px;margin:18px 0}.chapter{border-left:5px solid var(--brand)}.section{margin-left:12px;border-left:4px solid #a78bfa}.case-card{margin-left:24px;padding:0;overflow:hidden}.case-spec{padding:26px}.case-titlebar{display:flex;gap:12px;align-items:center;justify-content:space-between}.case-result{padding:24px;border-top:1px solid var(--line);background:#fcfcfd}.status-border-pass{border-left:5px solid var(--pass)}.status-border-fail{border-left:5px solid var(--fail)}.status-border-blocked{border-left:5px solid var(--blocked)}.status-border-not-run,.status-border-skipped{border-left:5px solid var(--not)}h1,h2,h3,h4,h5{line-height:1.25;scroll-margin-top:20px}h2{margin-top:30px}h3{margin-top:24px}a{color:#4f46e5}.anchor{scroll-margin-top:18px}.status{display:inline-flex;align-items:center;border-radius:999px;padding:3px 9px;font-size:11px;font-weight:750;letter-spacing:.03em}.pass{color:var(--pass);background:var(--pass-bg)}.fail{color:var(--fail);background:var(--fail-bg)}.blocked{color:var(--blocked);background:var(--blocked-bg)}.not-run,.skipped{color:var(--not);background:var(--not-bg)}.summary-grid{display:grid;grid-template-columns:repeat(5,minmax(90px,1fr));gap:12px;margin:18px 0}.summary-card{padding:15px;border:1px solid var(--line);border-radius:11px;background:#fff}.summary-card strong{display:block;font-size:25px}.summary-card span{color:var(--muted);font-size:12px}.table-wrap{overflow:auto}table{border-collapse:collapse;width:100%;margin:12px 0}th,td{border:1px solid var(--line);padding:9px 11px;text-align:left;vertical-align:top}th{background:#f8fafc}code{font-family:"SFMono-Regular",Consolas,monospace;font-size:.9em;background:#f2f4f7;border-radius:5px;padding:1px 4px}pre{overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere;background:#101828;color:#e5e7eb;border-radius:10px;padding:14px;font:12px/1.55 "SFMono-Regular",Consolas,monospace}pre code{background:none;padding:0}.json{background:#172033}.stderr{border-left:4px solid #f04438}.stdout{border-left:4px solid #12b76a}details{border:1px solid var(--line);border-radius:10px;margin:10px 0;background:#fff}summary{cursor:pointer;padding:10px 13px;font-weight:650;list-style:none}summary::-webkit-details-marker{display:none}.chevron{display:inline-block;margin-right:7px;transition:transform .18s}details[open]>summary .chevron{transform:rotate(90deg)}details>pre,details>.json,details>.command-body,details>.assertion-grid,details>.attachment-meta{margin:0 12px 12px}.case-result-head,.step-head{display:flex;align-items:center;justify-content:space-between;gap:12px}.case-result-head h3,.step-head h4{margin:0}.step{border-top:1px solid var(--line);padding:18px 0}.expected-observed{display:grid;grid-template-columns:1fr 1fr;gap:14px}.expected-observed h5{margin:8px 0}.command summary code{color:#344054}.exit-code{float:right;color:var(--muted);font-size:11px}.assertion-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:10px}.assertion{border:1px solid var(--line);border-radius:8px;padding:10px}.attachment-image{display:block;max-width:100%;max-height:700px;border:1px solid var(--line);border-radius:8px}.attachment-meta{font-size:12px;color:var(--muted)}.download{display:inline-block;padding:8px 12px;background:var(--brand);color:white;border-radius:7px;text-decoration:none}.reason{background:var(--fail-bg);border:1px solid #fecdca;padding:12px;border-radius:9px}.remarks{background:#f9fafb;padding:12px;border-radius:8px}.hidden{display:none!important}.toolbar{display:flex;gap:8px;justify-content:flex-end;margin:12px 0}.button{border:1px solid var(--line);background:#fff;border-radius:8px;padding:7px 10px;cursor:pointer}.button:hover{background:#f8fafc}.source-json{max-height:500px}@media(max-width:900px){.layout{display:block}.sidebar{position:relative;height:auto}.main{padding:20px}.summary-grid{grid-template-columns:repeat(2,1fr)}.expected-observed{grid-template-columns:1fr}.case-card,.section{margin-left:0}}@media print{.sidebar,.toolbar{display:none}.layout{display:block}.main{max-width:none;padding:0}.case-card{break-inside:avoid}details>*{display:block!important}} +""" + +JS = r""" +(()=>{const q=s=>document.querySelector(s),qa=s=>[...document.querySelectorAll(s)];let filter='ALL';function apply(){const term=(q('#search').value||'').toLowerCase();qa('.case-card').forEach(c=>{const okStatus=filter==='ALL'||c.dataset.status===filter;const okText=!term||c.textContent.toLowerCase().includes(term);c.classList.toggle('hidden',!(okStatus&&okText));});qa('.toc-case').forEach(a=>{const target=q(a.getAttribute('href'));a.classList.toggle('hidden',!target||target.classList.contains('hidden'));});}q('#search').addEventListener('input',apply);qa('.filter').forEach(b=>b.addEventListener('click',()=>{qa('.filter').forEach(x=>x.classList.remove('active'));b.classList.add('active');filter=b.dataset.status;apply();}));q('#expand-all').addEventListener('click',()=>qa('details').forEach(d=>d.open=true));q('#collapse-all').addEventListener('click',()=>qa('details').forEach(d=>d.open=false));})(); +""" + + +def render_report( + plan: Plan, run: dict[str, Any], results: dict[str, tuple[dict[str, Any], Path]] +) -> str: + title = str(plan.index.get("title", plan.index["id"])) + run_id = str(run["id"]) + summary = run["summary"]["by_status"] + body: list[str] = [ + f'

      {html.escape(title)}

      Run {html.escape(run_id)} · {html.escape(str(run.get("started_at", "")))} → {html.escape(str(run.get("finished_at", "")))}

      ' + ] + body.append('

      Run summary

      ') + for status in STATUS: + body.append( + f'
      {status_badge(status)}{summary[status]["count"]}test cases
      ' + ) + body.append( + '
      › Software under test' + + json_block(run.get("software_under_test", {})) + + '
      › Environment and executor' + + json_block( + { + "environment": run.get("environment", {}), + "executor": run.get("executor", {}), + } + ) + + "
      " + ) + body.append( + '

      Test guide

      ' + + markdown_to_html(plan.guide_path.read_text(encoding="utf-8")) + + "
      " + ) + current_chapter = current_section = None + for case in plan.cases: + if case.chapter_id != current_chapter: + if current_section is not None: + body.append("
") + if current_chapter is not None: + body.append("
") + body.append( + f'

{html.escape(case.chapter_title)}

' + ) + if doc := plan.chapter_docs.get(case.chapter_id): + body.append(markdown_to_html(doc.read_text(encoding="utf-8"))) + current_chapter = case.chapter_id + current_section = None + if case.section_id != current_section: + if current_section is not None: + body.append("
") + body.append( + f'

{html.escape(case.section_title)}

' + ) + if doc := plan.section_docs.get(case.section_id): + body.append(markdown_to_html(doc.read_text(encoding="utf-8"))) + current_section = case.section_id + result, result_path = results[case.id] + status = str(result["status"]) + body.append( + f'
{html.escape(case.priority)}

{html.escape(case.id)} · {html.escape(case.title)}

{status_badge(status)}
' + ) + chips = case.requirements + case.risks + case.tags + if chips: + body.append( + '

' + + " ".join(f"{html.escape(x)}" for x in chips) + + "

" + ) + body.append( + markdown_to_html(case.spec_path.read_text(encoding="utf-8")) + "
" + ) + body.append(render_case_result(case, result, result_path, plan) + "
") + if current_section is not None: + body.append("
") + if current_chapter is not None: + body.append("") + body.append( + '

Raw run JSON

› Show source
'
+        + html.escape(json.dumps(run, ensure_ascii=False, indent=2))
+        + "
" + ) + nav = nav_html(plan, results) + filters = ( + '
' + + "".join( + f'' + for s in ("ALL",) + STATUS + ) + + "
" + ) + return f"""{html.escape(title)} · {html.escape(run_id)}
{"".join(body)}
""" + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--plan", + type=Path, + required=True, + help="test-plan directory containing index.json", + ) + parser.add_argument("--run-id", required=True, help="run ID under plan/results") + parser.add_argument("--output", type=Path, help="self-contained HTML output path") + parser.add_argument( + "--validate-only", action="store_true", help="validate without rendering" + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv or sys.argv[1:]) + if not args.validate_only and args.output is None: + raise ReportError("--output is required unless --validate-only is used") + plan = load_plan(args.plan) + run, results = load_session_results(plan, args.run_id) + if args.validate_only: + print( + json.dumps( + { + "status": "valid", + "plan_id": plan.index["id"], + "run_id": args.run_id, + "cases": len(plan.cases), + }, + ensure_ascii=False, + ) + ) + return 0 + rendered = render_report(plan, run, results) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + print( + json.dumps( + { + "status": "rendered", + "output": str(args.output), + "bytes": len(rendered.encode()), + "cases": len(plan.cases), + }, + ensure_ascii=False, + ) + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ReportError as error: + print( + json.dumps({"status": "error", "message": str(error)}, ensure_ascii=False), + file=sys.stderr, + ) + raise SystemExit(2) diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md new file mode 100644 index 000000000..85df2f069 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md @@ -0,0 +1,20 @@ + +# TC-GW-PP-001: Forward a Proxy v1 address + + +## Objective + +Verify that a PP-enabled backend receives the declared source address. + + +## Steps + + +### Step 1: Query policy + +**Expected result:** port 8443 has `pp=true`. + + +### Step 2: Send request + +**Expected result:** the backend receives `198.51.100.27:45678`. diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/SHA256SUMS b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/SHA256SUMS new file mode 100644 index 000000000..0a95bd054 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/SHA256SUMS @@ -0,0 +1,5 @@ +bd09187e1cb77d4e60ea4772720d03dc9e783a51d8f943b5918d69fdac559b42 artifacts/backend-capture.json +d4bb03792a2c674dcdaac431a8ecacf66f49fb376bd5b268280716e12db88779 prompt.md +1d589f0c1d6f59e21c870a71d453876752b3fe732b6cd3a3aa8a8b3c959ccf97 result.json +deeff3680a68d0730ac41042d3e94b5d4cdcf6423f8c4d8f0e05038744f1650e runner.json +958eafa2fbb4d9f9308d08d3879c44a831fa791196ae8bb620c18472e2e53215 session.jsonl diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/artifacts/backend-capture.json b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/artifacts/backend-capture.json new file mode 100644 index 000000000..033008b13 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/artifacts/backend-capture.json @@ -0,0 +1 @@ +{"source":"198.51.100.27:45678","status":200} diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/prompt.md b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/prompt.md new file mode 100644 index 000000000..6b295becf --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/prompt.md @@ -0,0 +1 @@ +Run the sample case. diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/result.json b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/result.json new file mode 100644 index 000000000..24ff49af1 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/result.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "case_id": "tc-gw-pp-001", + "status": "PASS", + "summary": "The Proxy v1 source address reached the backend.", + "steps": [ + {"id": "tc-gw-pp-001-step-01", "status": "PASS", "observed": "Port 8443 had pp=true."}, + {"id": "tc-gw-pp-001-step-02", "status": "PASS", "observed": "Backend observed 198.51.100.27:45678 and returned HTTP 200."} + ], + "artifacts": [ + {"name": "Backend capture", "path": "artifacts/backend-capture.json"} + ], + "remarks": "" +} diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/runner.json b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/runner.json new file mode 100644 index 000000000..6b13c0c68 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/runner.json @@ -0,0 +1,15 @@ +{ + "schema_version": "1.0", + "run_id": "run-demo", + "case_id": "tc-gw-pp-001", + "agent": {"type": "codex", "model": "sample-model"}, + "session": {"format": "codex-jsonl", "path": "session.jsonl", "events": 7}, + "prompt_path": "prompt.md", + "result_path": "result.json", + "started_at": "2026-07-23T08:00:00.000Z", + "finished_at": "2026-07-23T08:00:10.000Z", + "duration_ms": 10000, + "exit_code": 0, + "result_valid": true, + "result_error": null +} diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/session.jsonl b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/session.jsonl new file mode 100644 index 000000000..f05ff50e3 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/session.jsonl @@ -0,0 +1,7 @@ +{"type":"thread.started","thread_id":"demo","model":"sample-model"} +{"type":"item.completed","item":{"type":"agent_message","text":"[DSTACK-TEST-STEP-START tc-gw-pp-001-step-01]"}} +{"type":"item.completed","item":{"type":"command_execution","command":"query-policy","aggregated_output":"{\"port\":8443,\"pp\":true}\n","exit_code":0}} +{"type":"item.completed","item":{"type":"agent_message","text":"[DSTACK-TEST-STEP-END tc-gw-pp-001-step-01 PASS]"}} +{"type":"item.completed","item":{"type":"agent_message","text":"[DSTACK-TEST-STEP-START tc-gw-pp-001-step-02]"}} +{"type":"item.completed","item":{"type":"command_execution","command":"send-request","aggregated_output":"HTTP 200 source=198.51.100.27:45678\n","exit_code":0}} +{"type":"item.completed","item":{"type":"agent_message","text":"[DSTACK-TEST-STEP-END tc-gw-pp-001-step-02 PASS]"}} diff --git a/tools/dstack-test/tests/fixtures/sample-plan/README.md b/tools/dstack-test/tests/fixtures/sample-plan/README.md new file mode 100644 index 000000000..babd28aef --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/README.md @@ -0,0 +1,4 @@ + +# Sample test guide + +This fixture demonstrates an offline dstack test report. diff --git a/tools/dstack-test/tests/fixtures/sample-plan/index.json b/tools/dstack-test/tests/fixtures/sample-plan/index.json new file mode 100644 index 000000000..5e1108f89 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/index.json @@ -0,0 +1,38 @@ +{ + "schema_version": "1.0", + "id": "plan-sample", + "title": "Sample Gateway Test Plan", + "guide": {"path": "README.md", "anchor": "sample-test-guide"}, + "chapters": [ + { + "id": "chapter-gateway", + "order": 1, + "title": "Gateway", + "path": "01-gateway", + "sections": [ + { + "id": "section-proxy-protocol", + "order": 1, + "title": "Proxy Protocol", + "path": "01-gateway/01-proxy-protocol", + "cases": [ + { + "id": "tc-gw-pp-001", + "order": 1, + "title": "Forward a Proxy v1 address", + "priority": "P0", + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001", + "spec": { + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md", + "anchor": "tc-gw-pp-001" + }, + "requirements": ["req-gw-pp-v1"], + "risks": ["risk-client-address-loss"], + "tags": ["gateway", "e2e"] + } + ] + } + ] + } + ] +} diff --git a/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/SHA256SUMS b/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/SHA256SUMS new file mode 100644 index 000000000..01f43ac95 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/SHA256SUMS @@ -0,0 +1 @@ +31433a80e8e7c0bccb35f138381b8f05a39806458f94d090d71d44cdab39e9da run.json diff --git a/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/run.json b/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/run.json new file mode 100644 index 000000000..f9d2acde2 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/run.json @@ -0,0 +1,31 @@ +{ + "schema_version": "1.0", + "id": "run-demo", + "anchor": "run-demo", + "plan_id": "plan-sample", + "status": "COMPLETED", + "started_at": "2026-07-23T08:00:00.000Z", + "finished_at": "2026-07-23T08:00:10.000Z", + "executors": [{"type": "codex", "model": "sample-model"}], + "software_under_test": {"git_revision": "demo"}, + "environment": {"level": "INTEGRATION"}, + "summary": { + "total": 1, + "completed": 1, + "by_status": { + "PASS": {"count": 1, "case_refs": ["#result-tc-gw-pp-001"]}, + "FAIL": {"count": 0, "case_refs": []}, + "BLOCKED": {"count": 0, "case_refs": []}, + "NOT_RUN": {"count": 0, "case_refs": []}, + "SKIPPED": {"count": 0, "case_refs": []} + } + }, + "case_results": [ + { + "id": "tc-gw-pp-001", + "anchor": "result-tc-gw-pp-001", + "status": "PASS", + "result_path": "../../01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/result.json" + } + ] +} diff --git a/tools/dstack-test/tests/test_dstack_test.py b/tools/dstack-test/tests/test_dstack_test.py new file mode 100644 index 000000000..d708ecb1d --- /dev/null +++ b/tools/dstack-test/tests/test_dstack_test.py @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: D100, D101, D102, D103 + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +TOOL_DIR = HERE.parent +FIXTURE = HERE / "fixtures" / "sample-plan" +CLI = TOOL_DIR / "dstack-test" +sys.path.insert(0, str(TOOL_DIR)) + +import dstack_test # noqa: E402 +import render # noqa: E402 + + +class DstackTestTests(unittest.TestCase): + def copy_fixture(self, root: Path) -> Path: + plan = root / "plan" + shutil.copytree(FIXTURE, plan) + return plan + + def test_validate_and_render_fixture(self) -> None: + plan = render.load_plan(FIXTURE) + valid = dstack_test.validate_run(plan, "run-demo") + self.assertEqual(valid["cases"], 1) + run, results = render.load_session_results(plan, "run-demo") + output = render.render_report(plan, run, results) + self.assertIn("Sample Gateway Test Plan", output) + self.assertIn("Complete agent session (7 events)", output) + self.assertIn("198.51.100.27:45678", output) + self.assertIn("session-tc-gw-pp-001-event-2", output) + + def test_finalize_rebuilds_run_summary(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + plan_path = self.copy_fixture(Path(temporary)) + shutil.rmtree(plan_path / "results" / "run-demo") + plan = render.load_plan(plan_path) + run = dstack_test.finalize_run( + plan, "run-demo", {"environment": {"level": "INTEGRATION"}} + ) + self.assertEqual(run["status"], "COMPLETED") + self.assertEqual(run["summary"]["by_status"]["PASS"]["count"], 1) + self.assertEqual( + dstack_test.validate_run(plan, "run-demo")["status"], "valid" + ) + + def test_invalid_session_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + plan_path = self.copy_fixture(Path(temporary)) + session = ( + plan_path + / "01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/session.jsonl" + ) + session.write_text("not json\n", encoding="utf-8") + with self.assertRaises(dstack_test.DstackTestError): + dstack_test.validate_run(render.load_plan(plan_path), "run-demo") + + def test_run_case_with_fake_codex(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + plan = self.copy_fixture(root) + fake_bin = root / "bin" + fake_bin.mkdir() + fake = fake_bin / "codex" + fake.write_text( + """#!/usr/bin/env python3 +import json, pathlib, re, sys +prompt=sys.argv[-1] +match=re.search(r'原子写入([^\\n]+/result\\.json)',prompt) +if not match: raise SystemExit(9) +path=pathlib.Path(match.group(1).strip()) +path.parent.mkdir(parents=True,exist_ok=True) +result={ + 'schema_version':'1.0','case_id':'tc-gw-pp-001','status':'PASS','summary':'fake agent passed', + 'steps':[ + {'id':'tc-gw-pp-001-step-01','status':'PASS','observed':'policy matched'}, + {'id':'tc-gw-pp-001-step-02','status':'PASS','observed':'request matched'}], + 'artifacts':[],'remarks':''} +tmp=path.with_suffix('.tmp'); tmp.write_text(json.dumps(result)); tmp.replace(path) +print(json.dumps({'type':'thread.started','model':'fake-model'})) +print(json.dumps({'type':'item.completed','item':{'type':'agent_message','text':'tc-gw-pp-001-step-01 tc-gw-pp-001-step-02'}})) +""", + encoding="utf-8", + ) + fake.chmod(0o755) + env = os.environ.copy() + env["PATH"] = str(fake_bin) + os.pathsep + env["PATH"] + completed = subprocess.run( + [ + str(CLI), + "run-case", + "--agent", + "codex", + "--plan", + str(plan), + "--case", + "tc-gw-pp-001", + "--run-id", + "run-fake", + "--workdir", + str(root), + "--", + "fake execution", + ], + text=True, + capture_output=True, + env=env, + check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + result_dir = ( + plan / "01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-fake" + ) + self.assertEqual( + json.loads((result_dir / "result.json").read_text())["status"], "PASS" + ) + runner = json.loads((result_dir / "runner.json").read_text()) + self.assertEqual(runner["agent"]["model"], "fake-model") + self.assertTrue((result_dir / "session.jsonl").is_file()) + + def test_package_zip_is_self_contained(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + plan_path = self.copy_fixture(root) + old_run = plan_path / "results" / "run-old" + old_run.mkdir(parents=True) + (old_run / "must-not-be-packaged.txt").write_text("old") + output = root / "report.zip" + dstack_test.package_plan(render.load_plan(plan_path), "run-demo", output) + self.assertGreater(output.stat().st_size, 0) + with zipfile.ZipFile(output) as archive: + names = archive.namelist() + self.assertIn("plan/results/run-demo/run.json", names) + self.assertFalse(any("run-old" in name for name in names)) + + +if __name__ == "__main__": + unittest.main() From 9c1a7d440f5e773f13828909781c6bbdf9a3e3e7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 23 Jul 2026 02:46:00 -0700 Subject: [PATCH 2/8] docs(testing): use English for test specifications --- docs/testing/dstack-test-methodology.md | 256 +++++++--------- docs/testing/test-case-authoring-spec.md | 306 ++++++++------------ docs/testing/test-report-output-spec.md | 200 ++++++------- tools/dstack-test/dstack_test.py | 52 ++-- tools/dstack-test/tests/test_dstack_test.py | 2 +- 5 files changed, 343 insertions(+), 473 deletions(-) diff --git a/docs/testing/dstack-test-methodology.md b/docs/testing/dstack-test-methodology.md index 8a3c84c2c..ec95ab8fa 100644 --- a/docs/testing/dstack-test-methodology.md +++ b/docs/testing/dstack-test-methodology.md @@ -1,199 +1,137 @@ - -# dstack 测试方法 +# dstack Test Methodology + +This document defines the common process for dstack release testing, from change analysis and risk assessment through execution, evidence collection, and release decisions. See the [test-case authoring specification](test-case-authoring-spec.md#dstack-test-case-authoring-spec) and [report output specification](test-report-output-spec.md#dstack-test-report-output-spec) for normative formats. + + +## 1. Objectives + +Testing must produce reproducible, auditable, and traceable release evidence—not merely show that a script once exited successfully. A conclusion must be traceable from a requirement or risk to a case, step, original command evidence, observation, and attachment. + +Testing is complete only when: + +1. every relevant change, requirement, and material risk has explicit coverage; +2. an executor unfamiliar with the implementation can reproduce each case; +3. the native AI session preserves executed commands and their raw output; +4. simulated and physical-hardware results are reported separately; +5. tools can recompute aggregate status from atomic case results; and +6. references, attachment digests, and statistics are machine-verifiable. + + +## 2. Artifact layers + +Do not mix these four layers: + +| Layer | Purpose | Immutable after execution starts | +|---|---|---:| +| Change audit | Establishes changed behavior, dependencies, and risks | Yes | +| Test plan | Defines scope, topology, cases, and execution order | Yes | +| Case specification | Defines preconditions, actions, and expected results | Yes | +| `results//` | Records versions, native sessions, observations, and attachments | No, while running | -本文规定 dstack 版本测试从需求分析、风险建模、用例设计、环境执行、证据采集到发布判定的统一方法。具体用例格式见[测试用例编写规范](test-case-authoring-spec.md#dstack-test-case-authoring-spec)，执行产物格式见[测试报告输出规范](test-report-output-spec.md#dstack-test-report-output-spec)。 +A plan uses exactly three semantic levels: chapter, section, and case. Its machine-readable execution order is defined by `index.json`; its top-level `README.md` is the executor's environment guide. - -## 1. 目标 + +## 3. Workflow -测试应当提供可重复、可审计且可追踪的发布证据，而不只是证明某个脚本曾经退出为零。任一结论都应能从需求或风险追踪到用例、步骤、命令原文、观察结果和附件。 - -测试工作的完成条件是： - -1. 版本变化、产品需求和主要风险均有明确测试覆盖； -2. 测试规格可由不了解实现细节的执行者独立复现； -3. 每一步实际执行的命令、退出码、标准输出和标准错误均保存； -4. 模拟测试和真实硬件测试分别报告，不能互相替代； -5. 汇总状态可由工具从原子结果重新计算； -6. 报告中的引用、附件摘要和统计均可机器校验。 +### 3.1 Audit the release delta - -## 2. 测试资料分层 +Compare the previous released tag with the candidate commit. Inspect commits, pull requests, schemas, RPCs, command-line interfaces, configuration defaults, systemd units, image recipes, deployment manifests, migrations, and dependency changes. For every change record: -测试资料分为四类，禁止混写： +- the user-visible or operational behavior; +- affected components and interfaces; +- compatibility direction and version combinations; +- failure modes and security impact; +- the requirement and risk IDs used by test cases; and +- whether physical TEE hardware is required. -| 资料 | 作用 | 是否随执行变化 | -|---|---|---| -| 测试说明 `README.md` | 范围、环境配置、公共前置条件、执行与清理方法 | 否 | -| 测试索引 `index.json` | 章、节、用例顺序及机器可读引用 | 否 | -| 用例规格 `case.md` | 单条用例的目标、前置条件、数据、步骤和预期结果 | 否 | -| 执行结果 `results//` | 实际版本、命令证据、观察结果、附件和状态 | 是 | +Generated changelogs alone are insufficient. Follow data and control flow across component boundaries. -测试规格是只读输入。执行器只能在 `results/` 下创建当前 run 的产物，不得根据执行结果反向修改预期结果。 +### 3.2 Build a risk-based coverage matrix - -## 3. 组织结构 +Classify coverage as: -测试计划固定使用“章／节／用例”三个层级。每条用例必须拥有独立目录，便于附带 fixture、脚本和多次执行结果。 +- **new or changed functionality**: full positive, boundary, and relevant negative coverage; +- **regression**: behavior likely to be affected by shared code, configuration, images, protocols, or lifecycle changes; +- **compatibility**: supported mixed-version combinations and upgrade order; +- **security**: trust boundaries, identity, attestation, key handling, authorization, and secret disclosure; +- **operations**: install, upgrade, restart, recovery, logging, and diagnostics. -```text -/ -├── index.json -├── README.md -├── schemas/ -├── results//run.json -└── / - └──
/ - └── / - ├── case.md - ├── fixtures/ # 可选 - ├── scripts/ # 可选 - └── results// -``` +Prioritize by impact, likelihood, detectability, and breadth. `P0` covers release-blocking trust, data-loss, availability, or primary-path risks; `P1` covers important supported behavior; `P2` covers lower-risk variants. -目录名建议使用两位数字前缀维持人工浏览顺序；权威执行顺序仍来自 `index.json`，执行器不得通过文件名猜测顺序。 +### 3.3 Define environments - -## 4. 测试输入分析 - -在编写用例前，测试负责人应收集并冻结： +The plan guide must describe topology, component endpoints, credentials, test data, health checks, concurrency constraints, cleanup, and prohibited operations. Record common software versions once in run-level context. A case records a version override only when it deliberately uses a different component version. -- 上一个发布版本及本次候选 revision； -- change log、PR、schema、RPC 和配置变化； -- 产品需求及用户场景； -- 安全不变量和信任边界； -- 已知缺陷、迁移约束和运维方式； -- 可用的真实硬件、模拟器与外部服务。 +Environment levels are: -每项变化至少回答：谁使用它、正常行为是什么、错误输入如何处理、升级时如何兼容、失败后如何恢复、会影响哪些旧功能。 +- **UNIT**: isolated code-level validation; +- **SIMULATOR**: no-TEE or mock-attestation execution; +- **INTEGRATION**: deployed multi-component system; +- **HARDWARE**: physical supported TEE hardware. - -## 5. 追踪与覆盖 +Simulation may follow `docs/development-without-tee.md`; a local key provider that cannot start without TPM support may use TPM mode. Simulation never proves hardware-specific boot, measurement, attestation, sealing, or device behavior. Such unconfirmed items must be called out separately in the report. -每条用例应引用至少一个需求或风险。推荐维护以下关系： +### 3.4 Author and review cases -```text -Change / Requirement / Risk - ↓ - Test case - ↓ - Step - ↓ - Command and output - ↓ - Assertion/result -``` +Each case validates one independently decidable behavior and references at least one requirement or risk. Prefer three to eight logical steps. Every step defines an action and exact observable expected results. Do not write a separate failure criterion: any result that does not fully match the expected result is `FAIL`. -覆盖审计至少包含： +Review the plan for change coverage, regression breadth, compatibility matrices, security boundaries, operational recovery, test-data isolation, and cleanup before execution. -1. 新增和改变的功能； -2. 受影响功能的回归； -3. 新旧组件组合兼容； -4. 配置和 API 的合法、边界及错误输入； -5. 有状态组件的创建、运行、停止、重启、升级、回滚和恢复； -6. 安全策略的允许、拒绝及恢复； -7. 并发、隔离、幂等和重试； -8. 资源耗尽、依赖不可用和网络故障； -9. 可观测性、错误信息及审计记录。 +### 3.5 Execute -覆盖数量不是质量指标。一个包含多个独立断言的大用例应拆为若干原子用例，否则失败无法定位，结果也无法准确统计。 +An AI executor must: - -## 6. 环境等级 +1. read the plan `README.md` and `index.json`; +2. execute cases in index order unless the guide explicitly permits parallelism; +3. start a fresh Codex or Claude session for each case; +4. execute real commands rather than infer outcomes; +5. preserve the native JSONL session as step evidence; +6. write only a shallow atomic `result.json`; and +7. continue to later independent cases after a case-level failure. -执行报告必须标记环境等级： +The executor name and model are recorded by the runner. Secrets must never be emitted into sessions or artifacts. -| 等级 | 说明 | -|---|---| -| `UNIT` | 单进程或模块测试 | -| `INTEGRATION` | 多组件集成，但不使用模拟硬件证据代替真实平台 | -| `SIMULATED` | mock attestation、no-TEE、swtpm 等模拟环境 | -| `REAL_SGX` | 真实 SGX enclave | -| `REAL_TDX` | 真实 TDX CVM | -| `REAL_PLATFORM` | 指定 NVIDIA、GCP、AWS、SEV-SNP 等真实平台 | - -用例规格必须声明最低环境。若只在较低等级执行，原用例保持 `NOT_RUN` 或 `BLOCKED`，模拟结果应记录在单独用例中。 - - -## 7. 状态与判定 + +## 4. Status model -状态枚举固定为： - -- `PASS`：所有步骤的实际结果完全满足预期； -- `FAIL`：至少一步实际结果不完全满足预期； -- `BLOCKED`：外部环境或前置条件阻止被测行为开始； -- `NOT_RUN`：尚未执行； -- `SKIPPED`：预先定义的适用条件不成立。 - -单个用例不使用 `PARTIAL`。若一部分断言通过、一部分未执行，应拆分用例或将该用例判为非 `PASS`。顶层汇总必须由用例结果计算，不能手工覆盖。 - - -## 8. 标准执行流程 - -AI或人工执行器应按以下顺序工作： - -1. 阅读顶层 `README.md`； -2. 读取并校验 `index.json`、schema、路径、锚点和引用； -3. 创建唯一 `run_id`，记录执行器、公共版本和环境； -4. 按索引顺序逐章、逐节、逐用例读取 `case.md`； -5. 为每条用例启动独立AI会话，验证前置条件并执行各步骤； -6. runner将Agent原生JSONL事件流直接保存为`session.jsonl`，其中包含实际tool call、命令和原始输出； -7. 将观察结果与预期逐项比较，结束前原子写入浅层`result.json`； -8. 执行用例后置条件，记录未能清理的资源； -9. 全部完成后重新校验会话、结果、附件摘要、引用和统计； -10. 生成顶层`run.json`、`SHA256SUMS`，再打包上传或渲染HTML。 - -AI执行器必须记录实际提供方、产品和模型名称；无法从运行环境确认模型时填写 `unknown`，不得猜测。 +Case and step status is one of: - -## 9. 证据原则 +- `PASS`: every expected result was fully observed; +- `FAIL`: at least one expected result was not fully observed; +- `BLOCKED`: an external prerequisite prevented the tested behavior from starting; +- `NOT_RUN`: execution was not attempted; +- `SKIPPED`: omission was explicitly authorized and explained. -每一步至少包含一条实际执行的命令证据。命令和输出由Agent原生 -`session.jsonl`保存，不要求AI再次复制或转义。会话应包含： +`PARTIAL` is forbidden. A completed run may contain any terminal case status. A run is `INCOMPLETE` only when required case result artifacts are missing. -- 未经改写的命令字符串和工作目录； -- 已脱敏的环境变量； -- UTC开始、结束时间和退出码； -- stdout、stderr原文； -- 从原文提取的结构化 observed 值； -- 截图、日志或二进制捕获附件及 SHA-256。 +Product failure and test-infrastructure failure must be distinguished. A healthy system returning the wrong response is `FAIL`; an unavailable required laboratory host before the tested action begins is `BLOCKED`. -额外的截图、长日志或二进制捕获写入`artifacts/`并由`result.json`引用； -`result.json`只保存逐步观察总结，不嵌入大段输出。密钥、token、私钥和用户数据 -不得进入会话或附件；脱敏必须保留字段存在性和数据形状，且在报告中声明。 + +## 5. Evidence and traceability - -## 10. 版本与环境记录 +Every logical step must be supported by observed commands and raw output in the native session. Screenshots or other files are attachments, not replacements for command evidence where machine-readable evidence is available. Preserve timestamps, exit codes, stdout, stderr, and tool errors as supplied by the agent CLI. -公共被测版本只在 run 顶层记录一次，包括 Git revision、binary SHA-256、container digest 和 guest image digest。单个用例只在使用旧版或特殊构件时设置 `version_overrides`。 +Use explicit HTML anchors for all chapters, sections, cases, and steps. Do not rely on renderer-specific heading slugs. `index.json` is the authority for ordering and paths; IDs remain stable after publication. -环境记录至少包括：主机标识、内核、CPU架构、TEE能力、模拟标志、关键外部服务及配置摘要。路径和主机名如涉及隐私，可使用稳定别名，但必须足以复现拓扑。 + +## 6. Compatibility testing - -## 11. 发布判定 +Derive version combinations from supported deployment behavior rather than testing arbitrary permutations. For a rolling upgrade, cover at least: -发布结论应直接引用未通过的 P0/P1 用例及未覆盖需求。以下情况不得表述为“全量通过”： - -- P0 用例为 `FAIL`、`BLOCKED` 或 `NOT_RUN`； -- 真实硬件要求仅由模拟测试覆盖； -- 关键附件缺失或摘要不匹配； -- 结果引用失效； -- 执行偏离规格且未记录 deviation； -- 公共版本未固定到可追溯 digest。 +- latest control-plane services with both previous and latest guest images; +- persisted state created by the previous release and consumed by the candidate; +- protocol/schema defaults when one side omits newly introduced fields; +- upgrade order, restart behavior, and rollback where supported; and +- explicit rejection of unsupported combinations with actionable diagnostics. - -## 12. 打包和展示 +For dstack v0.6.0, the expected online topology includes latest VMM, KMS, and gateway components while instances may use a mixture of old and new images. -规范 Plan目录是平台和本地渲染器的唯一输入。推荐命令： + +## 7. Release decision -```bash -dstack-test render \ - --plan path/to/plan \ - --run-id run-20260723-001 \ - --output report.html -``` +The final report must provide coverage by requirement and risk, status counts, unresolved failures, blocked or skipped cases, simulation-only results, unconfirmed hardware items, and material deviations from the plan. Release acceptance criteria belong in the plan guide and must state which statuses or open risks block release. -渲染前必须完成结构、引用、状态、统计和附件摘要校验。生成的 HTML应内联样式、脚本、JSON、文本、图片和可下载附件，不依赖网络资源。 +Before publishing, run `dstack-test validate`, render the self-contained HTML report, and package the selected run. The package is an immutable review artifact and must not include secrets or results from unrelated run IDs. diff --git a/docs/testing/test-case-authoring-spec.md b/docs/testing/test-case-authoring-spec.md index d728aaaba..56330e759 100644 --- a/docs/testing/test-case-authoring-spec.md +++ b/docs/testing/test-case-authoring-spec.md @@ -1,113 +1,92 @@ - -# dstack 测试用例编写规范 +# dstack Test-Case Authoring Specification -本文规定测试 Plan、索引和 `case.md` 的写法。总体方法见[dstack 测试方法](dstack-test-methodology.md#dstack-test-methodology)，结果格式见[测试报告输出规范](test-report-output-spec.md#dstack-test-report-output-spec)。 +This document defines the normative layout and content of a dstack test plan and its `case.md` files. See the [test methodology](dstack-test-methodology.md#dstack-test-methodology) and [report output specification](test-report-output-spec.md#dstack-test-report-output-spec). - -## 1. Plan目录 + +## 1. Plan layout ```text / ├── index.json ├── README.md -├── schemas/ -├── results/ +├── results// └── / - ├── README.md # 可选 + ├── README.md # optional └──
/ - ├── README.md # 可选 + ├── README.md # optional └── / ├── case.md - ├── fixtures/ # 可选 - ├── scripts/ # 可选 - └── results/ + ├── fixtures/ # optional + ├── scripts/ # optional + └── results// ``` -测试组织只允许章、节、用例三个语义层级。用例必须放在独立目录中，规格文件固定命名为 `case.md`。 +Only chapter, section, and case are semantic organization levels. Each case has its own directory and a specification named `case.md`. + + +## 2. IDs and anchors - -## 2. ID和锚点 +All referenceable objects use explicit, stable, globally unique ASCII IDs. Use lowercase letters, digits, and hyphens. IDs must not change when titles change. Place `` before each referenceable heading and use relative paths with fragments for cross-file links. -所有可引用对象必须使用显式、稳定、全局唯一的 ASCII ID： +Recommended forms: ```text chapter-gateway section-gateway-proxy-protocol tc-gw-pp-001 tc-gw-pp-001-step-01 -req-gw-pp-inbound-v1 -risk-gw-pp-client-address +req-gw-pp-001 +risk-gw-spoofing-001 ``` -规则： - -- 仅使用小写字母、数字和连字符； -- ID发布后不得因标题变化而改变； -- Markdown不得依赖标题自动生成的 slug； -- 每个可引用标题前必须写 ``； -- 跨文件引用使用相对路径和 fragment，例如 `case.md#tc-gw-pp-001-step-01`。 + +## 3. Top-level guide - -## 3. 顶层说明书 +The top-level `README.md` is the first document an executor reads. It must state: -顶层 `README.md`是执行器必须首先阅读的文件，至少包含： +1. objectives and scope; +2. system topology and system under test; +3. hardware, software, account, and external-service requirements; +4. reproducible common setup commands; +5. shared preconditions and health checks; +6. status rules and release acceptance criteria; +7. evidence, redaction, and attachment rules; +8. order, concurrency, and stop conditions; +9. cleanup and recovery; and +10. validation, packaging, and rendering commands. -1. 测试目标和范围； -2. 被测系统和测试拓扑； -3. 硬件、软件、账号及外部服务要求； -4. 公共环境配置命令； -5. 公共前置条件和健康检查； -6. 状态判定规则； -7. 证据、脱敏和附件规则； -8. 执行顺序、并发限制和停止条件； -9. 清理与环境恢复方法； -10. 打包、上传和本地渲染方法。 +Avoid non-reproducible instructions such as “configure a working KMS.” -环境配置命令应可直接执行；不得使用“配置好KMS”“准备可用证书”等无法复现的描述。 - - + ## 4. `index.json` -索引是用例发现和顺序的权威来源。最低结构如下： +The index is authoritative for discovery and order. A minimal example is: ```json { "schema_version": "1.0", - "id": "plan-gateway-proxy-protocol", - "title": "Gateway Proxy Protocol Test Plan", - "guide": { - "path": "README.md", - "anchor": "test-plan-guide" - }, + "id": "dstack-v0-6-0-release", + "title": "dstack v0.6.0 Release Test Plan", + "guide": "README.md", "chapters": [ { "id": "chapter-gateway", - "order": 1, "title": "Gateway", - "path": "01-gateway", + "order": 1, "sections": [ { "id": "section-gateway-proxy-protocol", - "order": 1, "title": "Proxy Protocol", - "path": "01-gateway/01-proxy-protocol", + "order": 1, "cases": [ { "id": "tc-gw-pp-001", + "title": "Forward a Proxy v1 client address over TLS termination", "order": 1, - "title": "TLS终止路径转发Proxy v1地址", - "priority": "P0", - "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001", - "spec": { - "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md", - "anchor": "tc-gw-pp-001" - }, - "requirements": ["req-gw-pp-inbound-v1"], - "risks": ["risk-gw-pp-client-address"], - "tags": ["gateway", "proxy-protocol", "e2e"] + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001" } ] } @@ -117,184 +96,155 @@ risk-gw-pp-client-address } ``` -章、节和用例数组顺序必须与 `order`一致。路径必须留在 Plan根目录内，不得包含绝对路径或 `..`逃逸。 +Array order must agree with `order`. Paths must remain below the plan root and must not contain absolute paths or `..` traversal. - -## 5. `case.md`固定结构 + +## 5. Required `case.md` structure -每条用例按以下顺序编写： +Use this order: ```markdown -# TC-EXAMPLE-001：标题 +# TC-EXAMPLE-001: Title - -## 元数据 +## Metadata - -## 测试目标 +## Objective - -## 前置条件 +## Preconditions - -## 测试数据 +## Test Data - -## 测试步骤 +## Steps -### Step 1：步骤标题 +### Step 1: Step title -操作说明。 +Action instructions. -**预期结果：** +**Expected results:** -- 可观察结果一； -- 可观察结果二。 +- First observable result. +- Second observable result. - -## 后置条件 +## Postconditions ``` - -## 6. 元数据 - -元数据至少包含： + +## 6. Metadata -- ID； -- 优先级 `P0`、`P1`或`P2`； -- 类型，例如 Functional、Security、Compatibility、Regression、Performance； -- 最低环境等级； -- 是否适合自动化； -- Requirement和Risk引用； -- Tags。 +At minimum include case ID, priority (`P0`, `P1`, or `P2`), type (for example Functional, Security, Compatibility, Regression, or Performance), minimum environment level, automation suitability, and requirement/risk references. -公共版本不在每条用例重复。需要旧版或特殊组件时，在用例中增加“特殊版本要求”，执行结果写入 `version_overrides`。 +Do not repeat common versions in every case. When a case requires an old or special component version, add a **Special Version Requirements** field and record the actual value as a result-level version override. - -## 7. 测试目标 + +## 7. Objective -目标只描述一个可独立判定的行为。若标题中出现多个互不依赖的“并且”，通常应拆分用例。目标需要说明： +Define one independently decidable behavior: the relevant configuration or state, the action, and the essential externally observable result. Split a case when its title contains multiple independent “and” clauses. An implementation function name or a script's zero exit status is not a product objective. -- 什么配置或状态； -- 执行什么行为； -- 对外可观察的核心结果。 + +## 8. Preconditions -不得把实现函数名或已有测试脚本退出零当作产品目标。 +Preconditions must be verifiable and distinct from the tested action. Put shared environment conditions in the plan guide and only case-specific conditions in the case. If a prerequisite fails before the tested behavior starts, preserve evidence and report `BLOCKED`; do not report a product `FAIL` for setup failure. - -## 8. 前置条件 + +## 9. Test data -前置条件必须可验证并区分于测试步骤。每项应说明所需状态，而不是隐含测试结果。公共环境条件写在顶层说明书，用例只写本用例特有条件。 +Prefer JSON blocks for protocol fields, boundary values, and expected values. Document how random values are generated and preserve the actual values in results. Use RFC documentation address ranges for security-test addresses. Never put reusable credentials, tokens, private keys, or production secrets in plan files. -若前置条件失败，执行器记录命令证据并把用例标为 `BLOCKED`；不能把 setup错误记为产品 `FAIL`。 + +## 10. Steps and expected results - -## 9. 测试数据 +Prefer three to eight logical steps. One logical step may invoke several mechanical commands, but it validates one phase. Every step must have: -测试数据优先使用 JSON代码块，固定协议字段、边界值和预期值。随机数据必须说明生成方式并在结果中保存实际值。安全测试地址应使用 RFC文档网段，避免依赖执行主机随机地址。 +1. an explicit unique anchor; +2. a reproducible action; +3. precise, observable, comparable expected results; and +4. at least one item of command evidence in the native session. - -## 10. 步骤和预期结果 +Do not write a separate failure criterion. If the actual result does not fully satisfy the expected result, the step and case are `FAIL`. Replace vague words such as “normal,” “correct,” or “without errors” with exact states, fields, addresses, digests, counts, or response codes. -每条用例推荐 3至8个逻辑步骤。一个步骤可以执行多条机械命令，但只验证一个阶段性结果。每一步必须： + +## 11. Postconditions -1. 有显式锚点； -2. 描述执行动作； -3. 描述可观察、可比较的预期结果； -4. 能产生至少一条命令证据。 +State which data is removed, which services are stopped or retained, how modified policy/configuration is restored, which state is intentionally retained for later cases, and how cleanup failures are recorded. Cleanup failure does not erase the original test result but must be reported. -不单独编写失败判据：只要实际结果没有完全满足预期，该步骤即为 `FAIL`，用例即为 `FAIL`。 - -避免使用“正常”“正确”“无异常”等模糊结果，应写为具体状态、字段、地址、摘要、计数或响应码。 - - -## 11. 后置条件 - -后置条件描述： - -- 清除哪些测试数据； -- 停止或保留哪些服务； -- 如何恢复修改的 policy和配置； -- 哪些状态供后续用例复用； -- 清理失败时如何记录。 - - -## 12. Proxy Protocol样例 + +## 12. Proxy Protocol example ````markdown -# TC-GW-PP-001：启用PP的应用端口接收正确客户端地址 +# TC-GW-PP-001: A PP-enabled application port receives the client address + +## Metadata - -## 测试目标 +- Priority: P0 +- Type: Functional, Regression, Security +- Environment: INTEGRATION +- Requirements: req-gw-pp-001 +- Risks: risk-gw-spoofing-001 -验证Gateway启用Inbound Proxy Protocol且应用端口声明`pp=true`时， -上游Proxy v1客户端地址被转发给应用，后续HTTP请求正常完成。 +## Objective - -## 前置条件 +Verify that, when inbound Proxy Protocol is enabled and application port 8443 declares `pp=true`, gateway forwards the Proxy v1 client address to the application and completes the following HTTP request. -1. Gateway配置`inbound_pp_enabled=true`。 -2. guest已注册，端口8443配置`pp=true`。 -3. capture backend ready，Gateway已加载该instance的port policy。 +## Preconditions - -## 测试数据 +1. Gateway has `inbound_pp_enabled=true`. +2. The guest is registered and port 8443 has `pp=true`. +3. The capture backend is ready and gateway has loaded the instance port policy. + +## Test Data ```json { "source": "198.51.100.27:45678", - "destination": "203.0.113.10:443", - "app_port": 8443 + "destination": "203.0.113.10:8443", + "request_id": "tc-gw-pp-001" } ``` - -## 测试步骤 - -### Step 1：检查生效策略 +### Step 1: Check effective policy -查询guest和Gateway中的端口策略。 +Query the guest and gateway port policy. -**预期结果：** Instance ID正确，端口8443存在且`pp=true`。 +**Expected results:** The instance ID matches; port 8443 exists and has `pp=true`. -### Step 2：清理后端记录 +### Step 2: Reset capture state -清除capture backend历史记录。 +Clear earlier capture records. -**预期结果：** backend ready且记录数为0。 +**Expected results:** The backend is ready and contains zero records. -### Step 3：发送请求 +### Step 3: Send the request -发送Proxy v1 header，在同一连接完成TLS和HTTP请求。 +Send a Proxy v1 header, then complete TLS and HTTP on the same connection. -**预期结果：** TLS成功，HTTP返回200，request ID匹配。 +**Expected results:** TLS succeeds, HTTP returns 200, and the request ID matches. -### Step 4：检查捕获结果 +### Step 4: Inspect the capture -查询capture backend记录。 +Query the backend capture records. -**预期结果：** 恰好一条新记录，source和destination与测试数据一致，HTTP请求完整。 +**Expected results:** Exactly one new record exists; source and destination match the test data and the HTTP request is complete. ```` - -## 13. 评审清单 - -提交前确认： - -- 用例目录、ID和索引一致； -- 显式锚点存在且唯一； -- 每条用例引用需求或风险； -- 目标只有一个核心行为； -- 前置条件可验证； -- 每一步有精确预期； -- 步骤数量合理且能产生原始证据； -- 模拟和真实硬件要求明确； -- 后置条件可恢复环境； -- 不包含密钥、token和环境私密值。 + +## 13. Review checklist + +Before submission, confirm that: + +- directory, case ID, and index entry agree; +- explicit anchors are present and unique; +- each case references a requirement or risk; +- the objective has one core behavior; +- preconditions are verifiable; +- every step has precise expected results and raw evidence; +- step count is reasonable; +- simulation and physical-hardware requirements are explicit; +- postconditions restore the environment; and +- no secret or environment-private value is present. diff --git a/docs/testing/test-report-output-spec.md b/docs/testing/test-report-output-spec.md index 92c48e5b2..a9fb2e6fd 100644 --- a/docs/testing/test-report-output-spec.md +++ b/docs/testing/test-report-output-spec.md @@ -1,28 +1,27 @@ - -# dstack 测试报告输出规范 +# dstack Test Report Output Specification -本文规定 AI执行会话、用例总结、run汇总、附件、引用、打包和单文件HTML的格式。测试方法见[dstack 测试方法](dstack-test-methodology.md#dstack-test-methodology)，用例格式见[测试用例编写规范](test-case-authoring-spec.md#dstack-test-case-authoring-spec)。 +This document defines the normative format for AI sessions, case summaries, run summaries, attachments, cross-references, packages, and self-contained HTML reports. See the [test methodology](dstack-test-methodology.md#dstack-test-methodology) and [case authoring specification](test-case-authoring-spec.md#dstack-test-case-authoring-spec). -## 1. 输出原则 +## 1. Principles -1. 每条用例由一个独立Codex或Claude会话执行； -2. Agent原生JSONL会话是命令、tool call和原始输出的主要证据； -3. Agent只写一个浅层`result.json`，不在其中复制命令输出； -4. `runner.json`、run汇总、时间、退出码和摘要由工具生成； -5. 公共版本与环境只在run顶层记录； -6. 截图、日志和二进制文件作为独立附件； -7. 所有产物保留稳定锚点并可内联为单个HTML文件。 +1. Each case runs in an independent Codex or Claude session. +2. The agent's native JSONL is the primary source for commands, tool calls, and raw output. +3. The agent writes one shallow `result.json`; it does not copy command output into that file. +4. The runner generates `runner.json`, timestamps, exit code, checksums, and run aggregates. +5. Common versions and environment information appear once at run level. +6. Screenshots, long logs, and binary captures are separate attachments. +7. Stable anchors make every object linkable in one offline HTML report. - -## 2. 统一命令 + +## 2. Command interface -工具对外统一命名为`dstack-test`： +The single public command is `dstack-test`, with consistently named subcommands: -```bash +```text dstack-test run-case dstack-test run-plan dstack-test finalize @@ -31,25 +30,25 @@ dstack-test package dstack-test render ``` -参数统一使用kebab-case。Agent通过`--agent codex`或`--agent claude`选择，不使用独立的`--codex`、`--claude`开关。 +Options use kebab-case. Select an executor with `--agent codex` or `--agent claude`; do not introduce separate `--codex` or `--claude` switches. -## 3. 单用例执行 +## 3. Case execution ```bash dstack-test run-case \ --agent codex \ - --plan path/to/plan \ - --case tc-gw-pp-001 \ - --run-id run-20260723-001 \ - --workdir path/to/repository \ - -- "附加执行要求" + --plan \ + --case \ + --run-id \ + --workdir \ + -- "Additional execution constraints" ``` -runner自动把顶层说明书、`case.md`、结果目录、统一状态规则和总结schema加入Agent prompt。Agent必须先读说明书，再按用例执行；不得修改Plan规格。 +The runner supplies the plan guide, `case.md`, output location, status rules, and result schema in the prompt. The agent must read the guide before the case and must not modify plan specifications. -## 4. 结果目录 +## 4. Result layout ```text /results// @@ -62,92 +61,81 @@ runner自动把顶层说明书、`case.md`、结果目录、统一状态规则 └── SHA256SUMS /results// -├── context.json # 可选 ├── run.json +├── context.json # optional └── SHA256SUMS ``` -不同用例写不同目录，因此可以并行执行。已存在的非空run目录默认不得覆盖；显式重跑需使用`--overwrite`。 +Separate case directories permit independent sessions. A non-empty result directory must not be overwritten unless the caller explicitly supplies `--overwrite`. - + ## 5. `session.jsonl` -runner直接保存Agent CLI的原生事件流： - -- Codex：`codex exec --json`； -- Claude：`claude --print --verbose --output-format stream-json`。 - -每行必须是一个完整JSON对象。runner不改写或摘要stdout中的事件。命令、tool call、tool result、Agent消息和错误均保留在会话中。Claude和Codex格式不同，由HTML渲染器的adapter在读取时统一展示，原文件保持不变。 +The runner stores the native agent CLI event stream without rewriting it. Every non-empty line is one complete JSON object. Commands, tool calls, tool results, agent messages, and errors remain in their original format. Renderer adapters normalize Codex and Claude events only for display; the stored file remains unchanged. -Agent应在步骤开始和结束的消息中包含完整step ID，例如： +The agent should include the complete step ID when beginning and completing a step: ```text -[DSTACK-TEST-STEP-START tc-gw-pp-001-step-01] -[DSTACK-TEST-STEP-END tc-gw-pp-001-step-01 PASS] +tc-gw-pp-001-step-01 ``` -渲染器据此链接步骤与会话事件。没有标记时，可搜索step ID；仍找不到时显示`WHOLE_SESSION_FALLBACK`，并保留整个会话作为证据。 +The renderer links matching session events to the step. If no marker is found, it exposes the complete session as fallback evidence rather than inventing a narrower association. ## 6. `runner.json` -`runner.json`只能由`dstack-test`生成： +Only `dstack-test` writes this file: ```json { "schema_version": "1.0", "run_id": "run-20260723-001", "case_id": "tc-gw-pp-001", - "agent": { - "type": "codex", - "model": "gpt-5.4" - }, + "agent": {"type": "codex", "model": "gpt-5-codex"}, "session": { "format": "codex-jsonl", "path": "session.jsonl", - "events": 137 + "events": 42 }, "prompt_path": "prompt.md", "result_path": "result.json", - "started_at": "2026-07-23T08:10:00.000Z", - "finished_at": "2026-07-23T08:17:15.000Z", - "duration_ms": 435000, + "started_at": "2026-07-23T18:00:00.000Z", + "finished_at": "2026-07-23T18:04:32.000Z", + "duration_ms": 272000, "exit_code": 0, "result_valid": true, "result_error": null } ``` -模型名优先从会话事件提取，其次使用显式`--model`，否则填写`unknown`，不得猜测。 - -runner退出码描述Agent执行基础设施，不等于测试状态。Agent正常完成且写出合法`FAIL`结果时，runner仍可成功退出。 +Extract the model name from the session when possible, otherwise use explicit `--model`, then `unknown`; never guess. The exit code describes agent infrastructure, not product status. A valid `FAIL` result may accompany a successful agent exit. - -## 7. 浅层 `result.json` + +## 7. Shallow `result.json` -Agent在结束前原子写入： +Before exiting, the agent atomically writes: ```json { "schema_version": "1.0", "case_id": "tc-gw-pp-001", "status": "PASS", - "summary": "Gateway解析入站Proxy v1地址并向启用PP的应用端口正确转发。", + "summary": "Gateway forwarded the Proxy v1 address to the PP-enabled application port.", "steps": [ { "id": "tc-gw-pp-001-step-01", "status": "PASS", - "observed": "Gateway缓存中的端口8443配置为pp=true。" + "observed": "Gateway's cached port 8443 policy had pp=true." }, { "id": "tc-gw-pp-001-step-02", "status": "PASS", - "observed": "后端记录已清空，初始记录数为0。" + "observed": "The capture backend was ready and initially contained zero records." } ], "artifacts": [ { - "name": "backend-capture", + "name": "Backend capture", "path": "artifacts/backend-capture.json" } ], @@ -155,46 +143,42 @@ Agent在结束前原子写入： } ``` -约束： +Constraints: -- 状态只能是`PASS`、`FAIL`、`BLOCKED`、`NOT_RUN`或`SKIPPED`； -- 不得使用`PARTIAL`； -- `PASS`要求至少一个步骤，且所有步骤均为`PASS`； -- step ID必须来自`case.md`； -- `observed`只写简短观察总结，不复制命令原文； -- artifact路径必须位于本次结果目录内； -- 不能嵌入token、私钥或密钥材料。 +- status is `PASS`, `FAIL`, `BLOCKED`, `NOT_RUN`, or `SKIPPED`; +- `PARTIAL` is forbidden; +- `PASS` requires at least one step and all steps must be `PASS`; +- step IDs must come from `case.md`; +- `observed` is a concise observation, not copied raw command output; +- artifact paths must remain inside the result directory; and +- results must not contain tokens, private keys, or other secrets. -## 8. 附件 - -截图、长日志、JSON响应和二进制捕获写入`artifacts/`。`result.json`只保存名称和相对路径。finalize阶段生成`SHA256SUMS`；HTML根据MIME类型： +## 8. Attachments -- 图片：内联展示并提供下载； -- 文本和JSON：默认折叠展示原文； -- 其他二进制：内联为data URI下载链接。 - -附件不能使用绝对路径或`..`逃逸结果目录。 +Put screenshots, long logs, JSON responses, and binary captures under `artifacts/`; reference them by name and relative path in `result.json`. Generate `SHA256SUMS` during finalization. The renderer displays images inline, shows text and JSON in collapsible blocks, and provides embedded download links for other files. Absolute paths and `..` traversal are forbidden. -## 9. `run.json` +## 9. Run summary -全部用例结束后，`dstack-test finalize`扫描各用例结果并生成： +After all cases, `dstack-test finalize` scans case outputs and generates `run.json`: ```json { "schema_version": "1.0", "id": "run-20260723-001", "anchor": "run-20260723-001", - "plan_id": "plan-gateway-proxy-protocol", + "plan_id": "dstack-v0-6-0-release", "status": "COMPLETED", - "started_at": "2026-07-23T08:10:00.000Z", - "finished_at": "2026-07-23T09:12:00.000Z", - "executors": [ - {"type": "codex", "model": "gpt-5.4"} - ], - "software_under_test": {}, - "environment": {}, + "started_at": "2026-07-23T18:00:00.000Z", + "finished_at": "2026-07-23T20:00:00.000Z", + "executors": [{"type": "codex", "model": "gpt-5-codex"}], + "software_under_test": { + "repository": "Dstack-TEE/dstack", + "candidate": "0123456789abcdef", + "previous_release": "v0.5.11" + }, + "environment": {"level": "INTEGRATION", "simulated": true}, "summary": { "total": 1, "completed": 1, @@ -217,28 +201,28 @@ Agent在结束前原子写入： } ``` -公共软件版本和环境由`--context `提供。只有特殊旧版组件写入单用例`version_overrides`。 +Supply common versions and environment data through `--context `. Put exceptional component versions in the relevant case result only. -## 10. 校验 +## 10. Validation ```bash dstack-test validate --plan --run-id ``` -至少校验： +Validation must cover at least: -1. Plan目录、ID、锚点和索引顺序； -2. `session.jsonl`每个非空行均为JSON对象； -3. `runner.json`和`result.json`存在且case/run ID一致； -4. 用例和步骤状态一致； -5. artifact路径合法且文件存在； -6. 顶层统计可由用例结果重新计算； -7. 完成run不存在缺失用例； -8. `SHA256SUMS`与文件内容一致。 +1. plan paths, IDs, anchors, and index order; +2. one JSON object per non-empty session line; +3. required runner/result files and matching case/run IDs; +4. consistent case and step statuses; +5. safe, existing artifact paths; +6. recomputable run statistics; +7. no missing case in a completed run; and +8. complete and correct `SHA256SUMS` files. -## 11. 打包 +## 11. Packaging ```bash dstack-test package \ @@ -247,11 +231,10 @@ dstack-test package \ --output -.tar.gz ``` -支持`.tar.gz`、`.tgz`、`.tar`和`.zip`。工具必须先校验，再打包完整Plan及指定run结果； -不得夹带同一Plan下其他历史run的结果。 +Supported formats are `.tar.gz`, `.tgz`, `.tar`, and `.zip`. Validate before packaging. Include the complete plan and selected run, but exclude every other historical run stored beside it. -## 12. 单文件HTML +## 12. Self-contained HTML ```bash dstack-test render \ @@ -260,15 +243,14 @@ dstack-test render \ --output report.html ``` -HTML必须离线可用并内联全部CSS、JavaScript、会话事件、文本、JSON、图片和可下载附件。展示内容包括： - -- 章／节／用例三级目录； -- 测试说明书和`case.md`原始要求； -- 公共版本、环境和Agent模型； -- 状态统计、搜索和状态过滤； -- 每条用例总结及逐步observed结果； -- step到对应会话事件的跳转； -- 原始Agent消息、tool call、命令输出和错误的交互式折叠； -- 所有附件； -- 完整原始会话折叠区； -- 稳定交叉引用锚点。 +The output must work offline and inline all CSS, JavaScript, session events, text, JSON, images, and downloadable attachments. It must provide: + +- chapter/section/case navigation; +- the original guide and `case.md` requirements; +- common versions, environment, and executor model; +- status summaries, search, and status filters; +- each case summary and step observation; +- step-to-session-event links; +- collapsible original agent messages, tool calls, command output, and errors; +- every attachment and the complete raw session; and +- stable cross-reference anchors. diff --git a/tools/dstack-test/dstack_test.py b/tools/dstack-test/dstack_test.py index 87a7f6e82..fa5ab5f07 100755 --- a/tools/dstack-test/dstack_test.py +++ b/tools/dstack-test/dstack_test.py @@ -157,11 +157,11 @@ def result_schema_text(case_id: str) -> str: "schema_version": "1.0", "case_id": "{case_id}", "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", - "summary": "简短总结", + "summary": "Concise result summary", "steps": [ - {{"id": "{case_id}-step-01", "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", "observed": "实际观察总结"}} + {{"id": "{case_id}-step-01", "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", "observed": "Concise observation"}} ], - "artifacts": [{{"name": "附件名称", "path": "artifacts/相对路径"}}], + "artifacts": [{{"name": "Artifact name", "path": "artifacts/relative-path"}}], "remarks": "" }}''' @@ -169,31 +169,31 @@ def result_schema_text(case_id: str) -> str: def build_prompt( plan: render.Plan, case: render.CaseEntry, result_dir: Path, user_prompt: str ) -> str: - return f"""你正在执行一个dstack规范测试用例。 - -测试计划说明：{plan.guide_path} -测试用例：{case.spec_path} -执行结果目录：{result_dir} -完整会话由外部runner自动保存，你不需要手工复制命令输出。 - -执行规则： -1. 先阅读测试计划说明，再阅读case.md。 -2. 不得修改测试计划、index.json、case.md、fixtures或测试脚本。 -3. 只能把测试产物写到本次结果目录或说明书明确允许的临时目录。 -4. 严格按case.md逐步执行；每一步开始和结束时，在回复中包含对应step ID，方便会话索引。 -5. 实际命令必须通过工具执行，不得用推测或总结代替执行。 -6. 只要实际结果没有完全满足预期，该步骤和用例就是FAIL。 -7. 前置环境阻止被测行为开始时为BLOCKED；不得使用PARTIAL。 -8. 不得把模拟结果当作真实硬件结果。 -9. 不得把token、私钥、密钥材料或其他secret输出到会话或附件。 -10. 截图、日志和其他附件写入{result_dir / "artifacts"}。 -11. 结束前把总结原子写入{result_dir / "result.json"}，不得写到其他位置。 -12. result.json必须是以下浅层结构，不要嵌入命令输出： + return f"""You are executing a test case under the dstack test specification. + +Test plan guide: {plan.guide_path} +Test case: {case.spec_path} +Result directory: {result_dir} +The external runner saves the complete session; do not manually copy command output. + +Execution rules: +1. Read the test plan guide before reading case.md. +2. Do not modify the plan, index.json, case.md, fixtures, or test scripts. +3. Write artifacts only to this result directory or a temporary location explicitly allowed by the guide. +4. Follow case.md step by step. Include the step ID in messages at the start and end of each step so the session can be indexed. +5. Execute actual commands with tools; do not substitute inference or summaries for execution. +6. If an actual result does not fully satisfy the expected result, mark the step and case FAIL. +7. Use BLOCKED when an external prerequisite prevents the tested behavior from starting. Never use PARTIAL. +8. Never present simulation results as physical-hardware results. +9. Do not emit tokens, private keys, key material, or other secrets into the session or attachments. +10. Write screenshots, logs, and other attachments to {result_dir / "artifacts"}. +11. Before finishing, atomically write the summary to {result_dir / "result.json"} and nowhere else. +12. result.json must use this shallow structure and must not embed command output: {result_schema_text(case.id)} -13. 最终回复必须说明用例状态和result.json路径。 +13. In the final response, state the case status and result.json path. -调用方附加要求： -{user_prompt.strip() or "无"} +Additional caller requirements: +{user_prompt.strip() or "None"} """ diff --git a/tools/dstack-test/tests/test_dstack_test.py b/tools/dstack-test/tests/test_dstack_test.py index d708ecb1d..e25712e64 100644 --- a/tools/dstack-test/tests/test_dstack_test.py +++ b/tools/dstack-test/tests/test_dstack_test.py @@ -77,7 +77,7 @@ def test_run_case_with_fake_codex(self) -> None: """#!/usr/bin/env python3 import json, pathlib, re, sys prompt=sys.argv[-1] -match=re.search(r'原子写入([^\\n]+/result\\.json)',prompt) +match=re.search(r'atomically write the summary to ([^\\n]+/result\\.json)',prompt) if not match: raise SystemExit(9) path=pathlib.Path(match.group(1).strip()) path.parent.mkdir(parents=True,exist_ok=True) From 68d9d3a4ddcb3e0273c7f0a554c9218482aa98ff Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 23 Jul 2026 02:51:51 -0700 Subject: [PATCH 3/8] refactor(testing): simplify test runner invocation --- docs/testing/test-report-output-spec.md | 7 +- tools/dstack-test/README.md | 9 +- tools/dstack-test/dstack-test | 651 +++++++++++++++++++- tools/dstack-test/dstack_test.py | 633 ------------------- tools/dstack-test/tests/test_dstack_test.py | 18 +- 5 files changed, 671 insertions(+), 647 deletions(-) delete mode 100755 tools/dstack-test/dstack_test.py diff --git a/docs/testing/test-report-output-spec.md b/docs/testing/test-report-output-spec.md index a9fb2e6fd..e681e407f 100644 --- a/docs/testing/test-report-output-spec.md +++ b/docs/testing/test-report-output-spec.md @@ -30,17 +30,18 @@ dstack-test package dstack-test render ``` -Options use kebab-case. Select an executor with `--agent codex` or `--agent claude`; do not introduce separate `--codex` or `--claude` switches. +Options use kebab-case. Execution defaults to Codex; select Claude with +`--agent claude`. Do not introduce separate `--codex` or `--claude` switches. +`run-case` and `run-plan` generate a unique run ID when `--run-id` is omitted. +Commands that operate on an existing run still require its ID. ## 3. Case execution ```bash dstack-test run-case \ - --agent codex \ --plan \ --case \ - --run-id \ --workdir \ -- "Additional execution constraints" ``` diff --git a/tools/dstack-test/README.md b/tools/dstack-test/README.md index fd2001bb4..bd3724073 100644 --- a/tools/dstack-test/README.md +++ b/tools/dstack-test/README.md @@ -12,16 +12,15 @@ self-contained interactive HTML report. ```bash tools/dstack-test/dstack-test run-case \ - --agent codex \ --plan path/to/plan \ --case tc-gw-pp-001 \ - --run-id run-20260723-001 \ --workdir "$PWD" \ -- "Do not restart the physical host" ``` -Use `--agent claude` for Claude Code. Additional CLI options may be passed with -repeatable `--agent-arg`. The runner writes only below: +The executor defaults to Codex; use `--agent claude` for Claude Code. The run ID +is generated when `--run-id` is omitted. Additional CLI options may be passed +with repeatable `--agent-arg`. The runner writes only below: ```text /results// @@ -38,9 +37,7 @@ repeatable `--agent-arg`. The runner writes only below: ```bash tools/dstack-test/dstack-test run-plan \ - --agent codex \ --plan path/to/plan \ - --run-id run-20260723-001 \ --context run-context.json \ -- "Follow the environment restrictions in README.md" ``` diff --git a/tools/dstack-test/dstack-test b/tools/dstack-test/dstack-test index 88478fef5..0c093118d 100755 --- a/tools/dstack-test/dstack-test +++ b/tools/dstack-test/dstack-test @@ -1,6 +1,649 @@ -#!/usr/bin/env bash +#!/usr/bin/env python3 # SPDX-FileCopyrightText: © 2026 Phala Network # SPDX-License-Identifier: Apache-2.0 -set -euo pipefail -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -exec python3 "$SCRIPT_DIR/dstack_test.py" "$@" +# ruff: noqa: D100, D101, D102, D103 +"""Run, validate, package, and render dstack test plans.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import secrets +import shutil +import subprocess +import sys +import tarfile +import tempfile +import time +import zipfile +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import render + +TERMINAL_STATUS = ("PASS", "FAIL", "BLOCKED", "NOT_RUN", "SKIPPED") + + +class DstackTestError(Exception): + """A user-facing dstack-test error.""" + + +def utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def default_run_id() -> str: + timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + return f"run-{timestamp}-{secrets.token_hex(3)}" + + +def atomic_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", dir=path.parent, delete=False + ) as output: + json.dump(value, output, ensure_ascii=False, indent=2) + output.write("\n") + temporary = Path(output.name) + temporary.replace(path) + + +def find_case(plan: render.Plan, value: str) -> render.CaseEntry: + candidate = Path(value).resolve() + matches = [ + case for case in plan.cases if case.id == value or case.path == candidate + ] + if len(matches) != 1: + raise DstackTestError( + f"expected one indexed case for {value!r}, found {len(matches)}" + ) + return matches[0] + + +def validate_summary(case: render.CaseEntry, path: Path) -> dict[str, Any]: + result = render.load_json(path) + if result.get("case_id") != case.id: + raise DstackTestError(f"result case_id must be {case.id!r}: {path}") + status = result.get("status") + if status not in TERMINAL_STATUS: + raise DstackTestError(f"invalid result status {status!r}: {path}") + if not isinstance(result.get("summary"), str) or not result["summary"].strip(): + raise DstackTestError(f"result summary must be non-empty: {path}") + steps = result.get("steps", []) + if not isinstance(steps, list): + raise DstackTestError(f"result steps must be an array: {path}") + seen: set[str] = set() + for step in steps: + if not isinstance(step, dict): + raise DstackTestError(f"invalid step in {path}") + step_id = step.get("id") + if not isinstance(step_id, str) or not step_id.startswith(case.id + "-step-"): + raise DstackTestError(f"invalid step ID {step_id!r} in {path}") + if step_id in seen: + raise DstackTestError(f"duplicate step ID {step_id!r} in {path}") + seen.add(step_id) + if step.get("status") not in TERMINAL_STATUS: + raise DstackTestError(f"invalid status for {step_id} in {path}") + if not isinstance(step.get("observed"), str) or not step["observed"].strip(): + raise DstackTestError(f"missing observed summary for {step_id} in {path}") + if status == "PASS" and ( + not steps or any(step["status"] != "PASS" for step in steps) + ): + raise DstackTestError( + f"PASS result requires one or more all-PASS steps: {path}" + ) + artifacts = result.get("artifacts", []) + if not isinstance(artifacts, list): + raise DstackTestError(f"artifacts must be an array: {path}") + for artifact in artifacts: + if not isinstance(artifact, dict) or not artifact.get("path"): + raise DstackTestError(f"invalid artifact in {path}") + artifact_path = render.safe_path( + path.parent, artifact["path"], "result artifact" + ) + try: + artifact_path.relative_to(path.parent.resolve()) + except ValueError as error: + raise DstackTestError( + f"artifact escapes result directory: {artifact_path}" + ) from error + if not artifact_path.is_file(): + raise DstackTestError(f"missing result artifact: {artifact_path}") + return result + + +def validate_session(path: Path) -> tuple[int, str | None]: + if not path.is_file() or path.stat().st_size == 0: + raise DstackTestError(f"missing or empty agent session: {path}") + count = 0 + model: str | None = None + with path.open(encoding="utf-8") as source: + for line_no, line in enumerate(source, 1): + if not line.strip(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError as error: + raise DstackTestError( + f"invalid session JSONL at {path}:{line_no}: {error}" + ) from error + if not isinstance(event, dict): + raise DstackTestError( + f"session event is not an object at {path}:{line_no}" + ) + count += 1 + model = model or extract_model(event) + if count == 0: + raise DstackTestError(f"agent session contains no events: {path}") + return count, model + + +def extract_model(value: Any) -> str | None: + if isinstance(value, dict): + for key in ("model", "model_name"): + candidate = value.get(key) + if isinstance(candidate, str) and candidate: + return candidate + for item in value.values(): + if found := extract_model(item): + return found + elif isinstance(value, list): + for item in value: + if found := extract_model(item): + return found + return None + + +def result_schema_text(case_id: str) -> str: + return f'''{{ + "schema_version": "1.0", + "case_id": "{case_id}", + "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", + "summary": "Concise result summary", + "steps": [ + {{"id": "{case_id}-step-01", "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", "observed": "Concise observation"}} + ], + "artifacts": [{{"name": "Artifact name", "path": "artifacts/relative-path"}}], + "remarks": "" +}}''' + + +def build_prompt( + plan: render.Plan, case: render.CaseEntry, result_dir: Path, user_prompt: str +) -> str: + return f"""You are executing a test case under the dstack test specification. + +Test plan guide: {plan.guide_path} +Test case: {case.spec_path} +Result directory: {result_dir} +The external runner saves the complete session; do not manually copy command output. + +Execution rules: +1. Read the test plan guide before reading case.md. +2. Do not modify the plan, index.json, case.md, fixtures, or test scripts. +3. Write artifacts only to this result directory or a temporary location explicitly allowed by the guide. +4. Follow case.md step by step. Include the step ID in messages at the start and end of each step so the session can be indexed. +5. Execute actual commands with tools; do not substitute inference or summaries for execution. +6. If an actual result does not fully satisfy the expected result, mark the step and case FAIL. +7. Use BLOCKED when an external prerequisite prevents the tested behavior from starting. Never use PARTIAL. +8. Never present simulation results as physical-hardware results. +9. Do not emit tokens, private keys, key material, or other secrets into the session or attachments. +10. Write screenshots, logs, and other attachments to {result_dir / "artifacts"}. +11. Before finishing, atomically write the summary to {result_dir / "result.json"} and nowhere else. +12. result.json must use this shallow structure and must not embed command output: +{result_schema_text(case.id)} +13. In the final response, state the case status and result.json path. + +Additional caller requirements: +{user_prompt.strip() or "None"} +""" + + +def agent_command( + agent: str, model: str | None, workdir: Path, prompt: str, extra: list[str] +) -> list[str]: + if agent == "codex": + command = ["codex", "exec", "--json", "--color", "never", "-C", str(workdir)] + if model: + command += ["--model", model] + command += extra + command += [prompt] + return command + if agent == "claude": + command = ["claude", "--print", "--verbose", "--output-format", "stream-json"] + if model: + command += ["--model", model] + command += extra + command += [prompt] + return command + raise DstackTestError(f"unsupported agent: {agent}") + + +def run_case( + plan: render.Plan, + case: render.CaseEntry, + run_id: str, + agent: str, + model: str | None, + workdir: Path, + prompt: str, + extra: list[str], + overwrite: bool, +) -> dict[str, Any]: + result_dir = case.path / "results" / run_id + if result_dir.exists() and any(result_dir.iterdir()) and not overwrite: + raise DstackTestError( + f"result directory is not empty; pass --overwrite to replace it: {result_dir}" + ) + if overwrite and result_dir.exists(): + shutil.rmtree(result_dir) + (result_dir / "artifacts").mkdir(parents=True, exist_ok=True) + full_prompt = build_prompt(plan, case, result_dir, prompt) + (result_dir / "prompt.md").write_text(full_prompt, encoding="utf-8") + session_path = result_dir / "session.jsonl" + stderr_path = result_dir / "agent-stderr.log" + started_at = utc_now() + start = time.monotonic_ns() + command = agent_command(agent, model, workdir, full_prompt, extra) + binary = shutil.which(command[0]) + if not binary: + raise DstackTestError(f"agent CLI not found: {command[0]}") + with session_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + completed = subprocess.run( + command, cwd=workdir, stdout=stdout, stderr=stderr, check=False + ) + duration_ms = (time.monotonic_ns() - start) // 1_000_000 + finished_at = utc_now() + event_count, detected_model = validate_session(session_path) + result_path = result_dir / "result.json" + result_valid = False + result_error = None + try: + result = validate_summary(case, result_path) + result_valid = True + except (DstackTestError, render.ReportError) as error: + result = None + result_error = str(error) + runner = { + "schema_version": "1.0", + "run_id": run_id, + "case_id": case.id, + "agent": {"type": agent, "model": detected_model or model or "unknown"}, + "session": { + "format": f"{agent}-jsonl", + "path": "session.jsonl", + "events": event_count, + }, + "prompt_path": "prompt.md", + "result_path": "result.json", + "started_at": started_at, + "finished_at": finished_at, + "duration_ms": duration_ms, + "exit_code": completed.returncode, + "result_valid": result_valid, + "result_error": result_error, + } + atomic_json(result_dir / "runner.json", runner) + write_sha256s(result_dir) + if completed.returncode != 0: + raise DstackTestError( + f"{agent} exited with {completed.returncode}; session preserved at {session_path}" + ) + if not result_valid: + raise DstackTestError( + f"agent completed without a valid result.json: {result_error}" + ) + return {"case": case.id, "status": result["status"], "result_dir": str(result_dir)} + + +def write_sha256s(root: Path) -> None: + entries: list[str] = [] + for path in sorted( + p for p in root.rglob("*") if p.is_file() and p.name != "SHA256SUMS" + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + entries.append(f"{digest} {path.relative_to(root).as_posix()}") + (root / "SHA256SUMS").write_text( + "\n".join(entries) + ("\n" if entries else ""), encoding="utf-8" + ) + + +def verify_sha256s(root: Path) -> None: + sums = root / "SHA256SUMS" + if not sums.is_file(): + raise DstackTestError(f"missing SHA256SUMS: {sums}") + expected_files: set[Path] = set() + for line_no, line in enumerate(sums.read_text(encoding="utf-8").splitlines(), 1): + match = re.fullmatch(r"([0-9a-f]{64}) (.+)", line) + if not match: + raise DstackTestError(f"invalid SHA256SUMS line at {sums}:{line_no}") + path = render.safe_path(root, match.group(2), "SHA256SUMS") + try: + path.relative_to(root.resolve()) + except ValueError as error: + raise DstackTestError( + f"SHA256SUMS path escapes result directory: {path}" + ) from error + if not path.is_file(): + raise DstackTestError(f"SHA256SUMS references missing file: {path}") + if hashlib.sha256(path.read_bytes()).hexdigest() != match.group(1): + raise DstackTestError(f"SHA-256 mismatch: {path}") + expected_files.add(path) + actual_files = { + path.resolve() + for path in root.rglob("*") + if path.is_file() and path.name != "SHA256SUMS" + } + if expected_files != actual_files: + missing = sorted( + str(path.relative_to(root)) for path in actual_files - expected_files + ) + raise DstackTestError(f"files missing from SHA256SUMS in {root}: {missing}") + + +def context_value(path: Path | None) -> dict[str, Any]: + if path is None: + return {} + value = render.load_json(path) + return value + + +def finalize_run( + plan: render.Plan, run_id: str, context: dict[str, Any] | None = None +) -> dict[str, Any]: + case_results: list[dict[str, Any]] = [] + by_status = {status: {"count": 0, "case_refs": []} for status in TERMINAL_STATUS} + started: list[str] = [] + finished: list[str] = [] + executors: list[dict[str, Any]] = [] + incomplete = False + for case in plan.cases: + result_dir = case.path / "results" / run_id + result_path = result_dir / "result.json" + runner_path = result_dir / "runner.json" + if not result_path.is_file() or not runner_path.is_file(): + incomplete = True + continue + result = validate_summary(case, result_path) + runner = render.load_json(runner_path) + validate_session(result_dir / "session.jsonl") + status = result["status"] + by_status[status]["count"] += 1 + by_status[status]["case_refs"].append(f"#result-{case.id}") + case_results.append( + { + "id": case.id, + "anchor": f"result-{case.id}", + "status": status, + "result_path": os.path.relpath( + result_path, plan.root / "results" / run_id + ), + } + ) + if runner.get("started_at"): + started.append(runner["started_at"]) + if runner.get("finished_at"): + finished.append(runner["finished_at"]) + if runner.get("agent") and runner["agent"] not in executors: + executors.append(runner["agent"]) + run_dir = plan.root / "results" / run_id + total = len(plan.cases) + run = { + "schema_version": "1.0", + "id": run_id, + "anchor": run_id, + "plan_id": plan.index["id"], + "status": "INCOMPLETE" + if incomplete or len(case_results) != total + else "COMPLETED", + "started_at": min(started) if started else None, + "finished_at": max(finished) if finished else None, + "executors": executors, + "software_under_test": (context or {}).get("software_under_test", {}), + "environment": (context or {}).get("environment", {}), + "summary": { + "total": total, + "completed": len(case_results), + "by_status": by_status, + }, + "case_results": case_results, + } + atomic_json(run_dir / "run.json", run) + if context: + atomic_json(run_dir / "context.json", context) + write_sha256s(run_dir) + return run + + +def validate_run(plan: render.Plan, run_id: str) -> dict[str, Any]: + run_path = plan.root / "results" / run_id / "run.json" + run = render.load_json(run_path) + if run.get("plan_id") != plan.index["id"] or run.get("id") != run_id: + raise DstackTestError("run/plan ID mismatch") + actual = {status: [] for status in TERMINAL_STATUS} + for item in run.get("case_results", []): + case = find_case(plan, str(item.get("id"))) + result_path = (run_path.parent / str(item.get("result_path", ""))).resolve() + try: + result_path.relative_to(plan.root) + except ValueError as error: + raise DstackTestError( + f"case result escapes plan root: {result_path}" + ) from error + result = validate_summary(case, result_path) + result_dir = result_path.parent + validate_session(result_dir / "session.jsonl") + runner = render.load_json(result_dir / "runner.json") + if not runner.get("result_valid"): + raise DstackTestError(f"runner marks invalid result for {case.id}") + verify_sha256s(result_dir) + actual[result["status"]].append(f"#result-{case.id}") + summary = run.get("summary", {}) + if summary.get("total") != len(plan.cases): + raise DstackTestError("summary total does not match plan") + for status in TERMINAL_STATUS: + item = summary.get("by_status", {}).get(status, {}) + if ( + item.get("count") != len(actual[status]) + or item.get("case_refs") != actual[status] + ): + raise DstackTestError(f"summary mismatch for {status}") + verify_sha256s(run_path.parent) + return { + "status": "valid", + "plan_id": plan.index["id"], + "run_id": run_id, + "cases": len(run.get("case_results", [])), + } + + +def package_plan(plan: render.Plan, run_id: str, output: Path) -> None: + validate_run(plan, run_id) + output.parent.mkdir(parents=True, exist_ok=True) + files = [] + for path in sorted(item for item in plan.root.rglob("*") if item.is_file()): + relative = path.relative_to(plan.root) + parts = relative.parts + # A package is one immutable execution report, not an archive of all + # historical executions that happen to live beside the plan. + if "results" in parts: + index = parts.index("results") + if len(parts) > index + 1 and parts[index + 1] != run_id: + continue + files.append((path, Path("plan") / relative)) + suffix = output.name.lower() + if suffix.endswith(".tar.gz") or suffix.endswith(".tgz"): + with tarfile.open(output, "w:gz") as archive: + for path, archive_path in files: + archive.add(path, arcname=archive_path, recursive=False) + elif suffix.endswith(".tar"): + with tarfile.open(output, "w") as archive: + for path, archive_path in files: + archive.add(path, arcname=archive_path, recursive=False) + elif suffix.endswith(".zip"): + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path, archive_path in files: + archive.write(path, archive_path) + else: + raise DstackTestError("package output must end in .tar.gz, .tgz, .tar, or .zip") + + +def add_common_plan( + parser: argparse.ArgumentParser, *, run_id_required: bool = True +) -> None: + parser.add_argument("--plan", type=Path, required=True, help="test-plan directory") + parser.add_argument( + "--run-id", + required=run_id_required, + default=None if run_id_required else default_run_id(), + help="shared run identifier" + + ("" if run_id_required else " (default: generated automatically)"), + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="dstack-test", description=__doc__) + sub = parser.add_subparsers(dest="subcommand", required=True) + run_case_parser = sub.add_parser( + "run-case", help="run one indexed case in a fresh AI session" + ) + add_common_plan(run_case_parser, run_id_required=False) + run_case_parser.add_argument( + "--case", required=True, help="case ID or indexed case directory" + ) + run_case_parser.add_argument( + "--agent", choices=("codex", "claude"), default="codex" + ) + run_case_parser.add_argument("--model") + run_case_parser.add_argument("--workdir", type=Path, default=Path.cwd()) + run_case_parser.add_argument("--prompt-file", type=Path) + run_case_parser.add_argument("--agent-arg", action="append", default=[]) + run_case_parser.add_argument("--overwrite", action="store_true") + run_case_parser.add_argument("prompt", nargs=argparse.REMAINDER) + + run_plan = sub.add_parser( + "run-plan", help="run every indexed case in its own AI session" + ) + add_common_plan(run_plan, run_id_required=False) + run_plan.add_argument("--agent", choices=("codex", "claude"), default="codex") + run_plan.add_argument("--model") + run_plan.add_argument("--workdir", type=Path, default=Path.cwd()) + run_plan.add_argument("--prompt-file", type=Path) + run_plan.add_argument( + "--context", + type=Path, + help="run-level JSON with software_under_test/environment", + ) + run_plan.add_argument("--agent-arg", action="append", default=[]) + run_plan.add_argument("--overwrite", action="store_true") + run_plan.add_argument("prompt", nargs=argparse.REMAINDER) + + finalize = sub.add_parser("finalize", help="materialize run.json from case results") + add_common_plan(finalize) + finalize.add_argument("--context", type=Path) + + validate = sub.add_parser("validate", help="validate a finalized test run") + add_common_plan(validate) + + package = sub.add_parser( + "package", help="validate and package a test plan and its results" + ) + add_common_plan(package) + package.add_argument("--output", type=Path, required=True) + + render_parser = sub.add_parser( + "render", help="render a validated run to one offline HTML file" + ) + add_common_plan(render_parser) + render_parser.add_argument("--output", type=Path, required=True) + return parser + + +def prompt_text(args: argparse.Namespace) -> str: + values = list(args.prompt) + if values and values[0] == "--": + values = values[1:] + text = " ".join(values) + if args.prompt_file: + file_text = args.prompt_file.read_text(encoding="utf-8") + text = f"{file_text}\n\n{text}" if text else file_text + return text + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + plan = render.load_plan(args.plan) + if args.subcommand == "run-case": + value = run_case( + plan, + find_case(plan, args.case), + args.run_id, + args.agent, + args.model, + args.workdir.resolve(), + prompt_text(args), + args.agent_arg, + args.overwrite, + ) + elif args.subcommand == "run-plan": + values = [] + for case in plan.cases: + try: + values.append( + run_case( + plan, + case, + args.run_id, + args.agent, + args.model, + args.workdir.resolve(), + prompt_text(args), + args.agent_arg, + args.overwrite, + ) + ) + except DstackTestError as error: + values.append({"case": case.id, "runner_error": str(error)}) + run = finalize_run(plan, args.run_id, context_value(args.context)) + value = {"cases": values, "run": run} + elif args.subcommand == "finalize": + value = finalize_run(plan, args.run_id, context_value(args.context)) + elif args.subcommand == "validate": + value = validate_run(plan, args.run_id) + elif args.subcommand == "package": + package_plan(plan, args.run_id, args.output) + value = { + "status": "packaged", + "output": str(args.output), + "sha256": hashlib.sha256(args.output.read_bytes()).hexdigest(), + } + elif args.subcommand == "render": + validate_run(plan, args.run_id) + run, results = render.load_session_results(plan, args.run_id) + rendered = render.render_report(plan, run, results) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + value = { + "status": "rendered", + "output": str(args.output), + "bytes": args.output.stat().st_size, + } + else: + raise AssertionError(args.subcommand) + print(json.dumps(value, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (DstackTestError, render.ReportError) as error: + print( + json.dumps({"status": "error", "message": str(error)}, ensure_ascii=False), + file=sys.stderr, + ) + raise SystemExit(2) diff --git a/tools/dstack-test/dstack_test.py b/tools/dstack-test/dstack_test.py deleted file mode 100755 index fa5ab5f07..000000000 --- a/tools/dstack-test/dstack_test.py +++ /dev/null @@ -1,633 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: © 2026 Phala Network -# SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: D100, D101, D102, D103 -"""Run, validate, package, and render dstack test plans.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -import re -import shutil -import subprocess -import sys -import tarfile -import tempfile -import time -import zipfile -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import render - -TERMINAL_STATUS = ("PASS", "FAIL", "BLOCKED", "NOT_RUN", "SKIPPED") - - -class DstackTestError(Exception): - """A user-facing dstack-test error.""" - - -def utc_now() -> str: - return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") - - -def atomic_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile( - "w", encoding="utf-8", dir=path.parent, delete=False - ) as output: - json.dump(value, output, ensure_ascii=False, indent=2) - output.write("\n") - temporary = Path(output.name) - temporary.replace(path) - - -def find_case(plan: render.Plan, value: str) -> render.CaseEntry: - candidate = Path(value).resolve() - matches = [ - case for case in plan.cases if case.id == value or case.path == candidate - ] - if len(matches) != 1: - raise DstackTestError( - f"expected one indexed case for {value!r}, found {len(matches)}" - ) - return matches[0] - - -def validate_summary(case: render.CaseEntry, path: Path) -> dict[str, Any]: - result = render.load_json(path) - if result.get("case_id") != case.id: - raise DstackTestError(f"result case_id must be {case.id!r}: {path}") - status = result.get("status") - if status not in TERMINAL_STATUS: - raise DstackTestError(f"invalid result status {status!r}: {path}") - if not isinstance(result.get("summary"), str) or not result["summary"].strip(): - raise DstackTestError(f"result summary must be non-empty: {path}") - steps = result.get("steps", []) - if not isinstance(steps, list): - raise DstackTestError(f"result steps must be an array: {path}") - seen: set[str] = set() - for step in steps: - if not isinstance(step, dict): - raise DstackTestError(f"invalid step in {path}") - step_id = step.get("id") - if not isinstance(step_id, str) or not step_id.startswith(case.id + "-step-"): - raise DstackTestError(f"invalid step ID {step_id!r} in {path}") - if step_id in seen: - raise DstackTestError(f"duplicate step ID {step_id!r} in {path}") - seen.add(step_id) - if step.get("status") not in TERMINAL_STATUS: - raise DstackTestError(f"invalid status for {step_id} in {path}") - if not isinstance(step.get("observed"), str) or not step["observed"].strip(): - raise DstackTestError(f"missing observed summary for {step_id} in {path}") - if status == "PASS" and ( - not steps or any(step["status"] != "PASS" for step in steps) - ): - raise DstackTestError( - f"PASS result requires one or more all-PASS steps: {path}" - ) - artifacts = result.get("artifacts", []) - if not isinstance(artifacts, list): - raise DstackTestError(f"artifacts must be an array: {path}") - for artifact in artifacts: - if not isinstance(artifact, dict) or not artifact.get("path"): - raise DstackTestError(f"invalid artifact in {path}") - artifact_path = render.safe_path( - path.parent, artifact["path"], "result artifact" - ) - try: - artifact_path.relative_to(path.parent.resolve()) - except ValueError as error: - raise DstackTestError( - f"artifact escapes result directory: {artifact_path}" - ) from error - if not artifact_path.is_file(): - raise DstackTestError(f"missing result artifact: {artifact_path}") - return result - - -def validate_session(path: Path) -> tuple[int, str | None]: - if not path.is_file() or path.stat().st_size == 0: - raise DstackTestError(f"missing or empty agent session: {path}") - count = 0 - model: str | None = None - with path.open(encoding="utf-8") as source: - for line_no, line in enumerate(source, 1): - if not line.strip(): - continue - try: - event = json.loads(line) - except json.JSONDecodeError as error: - raise DstackTestError( - f"invalid session JSONL at {path}:{line_no}: {error}" - ) from error - if not isinstance(event, dict): - raise DstackTestError( - f"session event is not an object at {path}:{line_no}" - ) - count += 1 - model = model or extract_model(event) - if count == 0: - raise DstackTestError(f"agent session contains no events: {path}") - return count, model - - -def extract_model(value: Any) -> str | None: - if isinstance(value, dict): - for key in ("model", "model_name"): - candidate = value.get(key) - if isinstance(candidate, str) and candidate: - return candidate - for item in value.values(): - if found := extract_model(item): - return found - elif isinstance(value, list): - for item in value: - if found := extract_model(item): - return found - return None - - -def result_schema_text(case_id: str) -> str: - return f'''{{ - "schema_version": "1.0", - "case_id": "{case_id}", - "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", - "summary": "Concise result summary", - "steps": [ - {{"id": "{case_id}-step-01", "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", "observed": "Concise observation"}} - ], - "artifacts": [{{"name": "Artifact name", "path": "artifacts/relative-path"}}], - "remarks": "" -}}''' - - -def build_prompt( - plan: render.Plan, case: render.CaseEntry, result_dir: Path, user_prompt: str -) -> str: - return f"""You are executing a test case under the dstack test specification. - -Test plan guide: {plan.guide_path} -Test case: {case.spec_path} -Result directory: {result_dir} -The external runner saves the complete session; do not manually copy command output. - -Execution rules: -1. Read the test plan guide before reading case.md. -2. Do not modify the plan, index.json, case.md, fixtures, or test scripts. -3. Write artifacts only to this result directory or a temporary location explicitly allowed by the guide. -4. Follow case.md step by step. Include the step ID in messages at the start and end of each step so the session can be indexed. -5. Execute actual commands with tools; do not substitute inference or summaries for execution. -6. If an actual result does not fully satisfy the expected result, mark the step and case FAIL. -7. Use BLOCKED when an external prerequisite prevents the tested behavior from starting. Never use PARTIAL. -8. Never present simulation results as physical-hardware results. -9. Do not emit tokens, private keys, key material, or other secrets into the session or attachments. -10. Write screenshots, logs, and other attachments to {result_dir / "artifacts"}. -11. Before finishing, atomically write the summary to {result_dir / "result.json"} and nowhere else. -12. result.json must use this shallow structure and must not embed command output: -{result_schema_text(case.id)} -13. In the final response, state the case status and result.json path. - -Additional caller requirements: -{user_prompt.strip() or "None"} -""" - - -def agent_command( - agent: str, model: str | None, workdir: Path, prompt: str, extra: list[str] -) -> list[str]: - if agent == "codex": - command = ["codex", "exec", "--json", "--color", "never", "-C", str(workdir)] - if model: - command += ["--model", model] - command += extra - command += [prompt] - return command - if agent == "claude": - command = ["claude", "--print", "--verbose", "--output-format", "stream-json"] - if model: - command += ["--model", model] - command += extra - command += [prompt] - return command - raise DstackTestError(f"unsupported agent: {agent}") - - -def run_case( - plan: render.Plan, - case: render.CaseEntry, - run_id: str, - agent: str, - model: str | None, - workdir: Path, - prompt: str, - extra: list[str], - overwrite: bool, -) -> dict[str, Any]: - result_dir = case.path / "results" / run_id - if result_dir.exists() and any(result_dir.iterdir()) and not overwrite: - raise DstackTestError( - f"result directory is not empty; pass --overwrite to replace it: {result_dir}" - ) - if overwrite and result_dir.exists(): - shutil.rmtree(result_dir) - (result_dir / "artifacts").mkdir(parents=True, exist_ok=True) - full_prompt = build_prompt(plan, case, result_dir, prompt) - (result_dir / "prompt.md").write_text(full_prompt, encoding="utf-8") - session_path = result_dir / "session.jsonl" - stderr_path = result_dir / "agent-stderr.log" - started_at = utc_now() - start = time.monotonic_ns() - command = agent_command(agent, model, workdir, full_prompt, extra) - binary = shutil.which(command[0]) - if not binary: - raise DstackTestError(f"agent CLI not found: {command[0]}") - with session_path.open("wb") as stdout, stderr_path.open("wb") as stderr: - completed = subprocess.run( - command, cwd=workdir, stdout=stdout, stderr=stderr, check=False - ) - duration_ms = (time.monotonic_ns() - start) // 1_000_000 - finished_at = utc_now() - event_count, detected_model = validate_session(session_path) - result_path = result_dir / "result.json" - result_valid = False - result_error = None - try: - result = validate_summary(case, result_path) - result_valid = True - except (DstackTestError, render.ReportError) as error: - result = None - result_error = str(error) - runner = { - "schema_version": "1.0", - "run_id": run_id, - "case_id": case.id, - "agent": {"type": agent, "model": detected_model or model or "unknown"}, - "session": { - "format": f"{agent}-jsonl", - "path": "session.jsonl", - "events": event_count, - }, - "prompt_path": "prompt.md", - "result_path": "result.json", - "started_at": started_at, - "finished_at": finished_at, - "duration_ms": duration_ms, - "exit_code": completed.returncode, - "result_valid": result_valid, - "result_error": result_error, - } - atomic_json(result_dir / "runner.json", runner) - write_sha256s(result_dir) - if completed.returncode != 0: - raise DstackTestError( - f"{agent} exited with {completed.returncode}; session preserved at {session_path}" - ) - if not result_valid: - raise DstackTestError( - f"agent completed without a valid result.json: {result_error}" - ) - return {"case": case.id, "status": result["status"], "result_dir": str(result_dir)} - - -def write_sha256s(root: Path) -> None: - entries: list[str] = [] - for path in sorted( - p for p in root.rglob("*") if p.is_file() and p.name != "SHA256SUMS" - ): - digest = hashlib.sha256(path.read_bytes()).hexdigest() - entries.append(f"{digest} {path.relative_to(root).as_posix()}") - (root / "SHA256SUMS").write_text( - "\n".join(entries) + ("\n" if entries else ""), encoding="utf-8" - ) - - -def verify_sha256s(root: Path) -> None: - sums = root / "SHA256SUMS" - if not sums.is_file(): - raise DstackTestError(f"missing SHA256SUMS: {sums}") - expected_files: set[Path] = set() - for line_no, line in enumerate(sums.read_text(encoding="utf-8").splitlines(), 1): - match = re.fullmatch(r"([0-9a-f]{64}) (.+)", line) - if not match: - raise DstackTestError(f"invalid SHA256SUMS line at {sums}:{line_no}") - path = render.safe_path(root, match.group(2), "SHA256SUMS") - try: - path.relative_to(root.resolve()) - except ValueError as error: - raise DstackTestError( - f"SHA256SUMS path escapes result directory: {path}" - ) from error - if not path.is_file(): - raise DstackTestError(f"SHA256SUMS references missing file: {path}") - if hashlib.sha256(path.read_bytes()).hexdigest() != match.group(1): - raise DstackTestError(f"SHA-256 mismatch: {path}") - expected_files.add(path) - actual_files = { - path.resolve() - for path in root.rglob("*") - if path.is_file() and path.name != "SHA256SUMS" - } - if expected_files != actual_files: - missing = sorted( - str(path.relative_to(root)) for path in actual_files - expected_files - ) - raise DstackTestError(f"files missing from SHA256SUMS in {root}: {missing}") - - -def context_value(path: Path | None) -> dict[str, Any]: - if path is None: - return {} - value = render.load_json(path) - return value - - -def finalize_run( - plan: render.Plan, run_id: str, context: dict[str, Any] | None = None -) -> dict[str, Any]: - case_results: list[dict[str, Any]] = [] - by_status = {status: {"count": 0, "case_refs": []} for status in TERMINAL_STATUS} - started: list[str] = [] - finished: list[str] = [] - executors: list[dict[str, Any]] = [] - incomplete = False - for case in plan.cases: - result_dir = case.path / "results" / run_id - result_path = result_dir / "result.json" - runner_path = result_dir / "runner.json" - if not result_path.is_file() or not runner_path.is_file(): - incomplete = True - continue - result = validate_summary(case, result_path) - runner = render.load_json(runner_path) - validate_session(result_dir / "session.jsonl") - status = result["status"] - by_status[status]["count"] += 1 - by_status[status]["case_refs"].append(f"#result-{case.id}") - case_results.append( - { - "id": case.id, - "anchor": f"result-{case.id}", - "status": status, - "result_path": os.path.relpath( - result_path, plan.root / "results" / run_id - ), - } - ) - if runner.get("started_at"): - started.append(runner["started_at"]) - if runner.get("finished_at"): - finished.append(runner["finished_at"]) - if runner.get("agent") and runner["agent"] not in executors: - executors.append(runner["agent"]) - run_dir = plan.root / "results" / run_id - total = len(plan.cases) - run = { - "schema_version": "1.0", - "id": run_id, - "anchor": run_id, - "plan_id": plan.index["id"], - "status": "INCOMPLETE" - if incomplete or len(case_results) != total - else "COMPLETED", - "started_at": min(started) if started else None, - "finished_at": max(finished) if finished else None, - "executors": executors, - "software_under_test": (context or {}).get("software_under_test", {}), - "environment": (context or {}).get("environment", {}), - "summary": { - "total": total, - "completed": len(case_results), - "by_status": by_status, - }, - "case_results": case_results, - } - atomic_json(run_dir / "run.json", run) - if context: - atomic_json(run_dir / "context.json", context) - write_sha256s(run_dir) - return run - - -def validate_run(plan: render.Plan, run_id: str) -> dict[str, Any]: - run_path = plan.root / "results" / run_id / "run.json" - run = render.load_json(run_path) - if run.get("plan_id") != plan.index["id"] or run.get("id") != run_id: - raise DstackTestError("run/plan ID mismatch") - actual = {status: [] for status in TERMINAL_STATUS} - for item in run.get("case_results", []): - case = find_case(plan, str(item.get("id"))) - result_path = (run_path.parent / str(item.get("result_path", ""))).resolve() - try: - result_path.relative_to(plan.root) - except ValueError as error: - raise DstackTestError( - f"case result escapes plan root: {result_path}" - ) from error - result = validate_summary(case, result_path) - result_dir = result_path.parent - validate_session(result_dir / "session.jsonl") - runner = render.load_json(result_dir / "runner.json") - if not runner.get("result_valid"): - raise DstackTestError(f"runner marks invalid result for {case.id}") - verify_sha256s(result_dir) - actual[result["status"]].append(f"#result-{case.id}") - summary = run.get("summary", {}) - if summary.get("total") != len(plan.cases): - raise DstackTestError("summary total does not match plan") - for status in TERMINAL_STATUS: - item = summary.get("by_status", {}).get(status, {}) - if ( - item.get("count") != len(actual[status]) - or item.get("case_refs") != actual[status] - ): - raise DstackTestError(f"summary mismatch for {status}") - verify_sha256s(run_path.parent) - return { - "status": "valid", - "plan_id": plan.index["id"], - "run_id": run_id, - "cases": len(run.get("case_results", [])), - } - - -def package_plan(plan: render.Plan, run_id: str, output: Path) -> None: - validate_run(plan, run_id) - output.parent.mkdir(parents=True, exist_ok=True) - files = [] - for path in sorted(item for item in plan.root.rglob("*") if item.is_file()): - relative = path.relative_to(plan.root) - parts = relative.parts - # A package is one immutable execution report, not an archive of all - # historical executions that happen to live beside the plan. - if "results" in parts: - index = parts.index("results") - if len(parts) > index + 1 and parts[index + 1] != run_id: - continue - files.append((path, Path("plan") / relative)) - suffix = output.name.lower() - if suffix.endswith(".tar.gz") or suffix.endswith(".tgz"): - with tarfile.open(output, "w:gz") as archive: - for path, archive_path in files: - archive.add(path, arcname=archive_path, recursive=False) - elif suffix.endswith(".tar"): - with tarfile.open(output, "w") as archive: - for path, archive_path in files: - archive.add(path, arcname=archive_path, recursive=False) - elif suffix.endswith(".zip"): - with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: - for path, archive_path in files: - archive.write(path, archive_path) - else: - raise DstackTestError("package output must end in .tar.gz, .tgz, .tar, or .zip") - - -def add_common_plan(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--plan", type=Path, required=True, help="test-plan directory") - parser.add_argument("--run-id", required=True, help="shared run identifier") - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="dstack-test", description=__doc__) - sub = parser.add_subparsers(dest="subcommand", required=True) - run_case_parser = sub.add_parser( - "run-case", help="run one indexed case in a fresh AI session" - ) - add_common_plan(run_case_parser) - run_case_parser.add_argument( - "--case", required=True, help="case ID or indexed case directory" - ) - run_case_parser.add_argument("--agent", choices=("codex", "claude"), required=True) - run_case_parser.add_argument("--model") - run_case_parser.add_argument("--workdir", type=Path, default=Path.cwd()) - run_case_parser.add_argument("--prompt-file", type=Path) - run_case_parser.add_argument("--agent-arg", action="append", default=[]) - run_case_parser.add_argument("--overwrite", action="store_true") - run_case_parser.add_argument("prompt", nargs=argparse.REMAINDER) - - run_plan = sub.add_parser( - "run-plan", help="run every indexed case in its own AI session" - ) - add_common_plan(run_plan) - run_plan.add_argument("--agent", choices=("codex", "claude"), required=True) - run_plan.add_argument("--model") - run_plan.add_argument("--workdir", type=Path, default=Path.cwd()) - run_plan.add_argument("--prompt-file", type=Path) - run_plan.add_argument( - "--context", - type=Path, - help="run-level JSON with software_under_test/environment", - ) - run_plan.add_argument("--agent-arg", action="append", default=[]) - run_plan.add_argument("--overwrite", action="store_true") - run_plan.add_argument("prompt", nargs=argparse.REMAINDER) - - finalize = sub.add_parser("finalize", help="materialize run.json from case results") - add_common_plan(finalize) - finalize.add_argument("--context", type=Path) - - validate = sub.add_parser("validate", help="validate a finalized test run") - add_common_plan(validate) - - package = sub.add_parser( - "package", help="validate and package a test plan and its results" - ) - add_common_plan(package) - package.add_argument("--output", type=Path, required=True) - - render_parser = sub.add_parser( - "render", help="render a validated run to one offline HTML file" - ) - add_common_plan(render_parser) - render_parser.add_argument("--output", type=Path, required=True) - return parser - - -def prompt_text(args: argparse.Namespace) -> str: - values = list(args.prompt) - if values and values[0] == "--": - values = values[1:] - text = " ".join(values) - if args.prompt_file: - file_text = args.prompt_file.read_text(encoding="utf-8") - text = f"{file_text}\n\n{text}" if text else file_text - return text - - -def main(argv: list[str] | None = None) -> int: - args = build_parser().parse_args(argv) - plan = render.load_plan(args.plan) - if args.subcommand == "run-case": - value = run_case( - plan, - find_case(plan, args.case), - args.run_id, - args.agent, - args.model, - args.workdir.resolve(), - prompt_text(args), - args.agent_arg, - args.overwrite, - ) - elif args.subcommand == "run-plan": - values = [] - for case in plan.cases: - try: - values.append( - run_case( - plan, - case, - args.run_id, - args.agent, - args.model, - args.workdir.resolve(), - prompt_text(args), - args.agent_arg, - args.overwrite, - ) - ) - except DstackTestError as error: - values.append({"case": case.id, "runner_error": str(error)}) - run = finalize_run(plan, args.run_id, context_value(args.context)) - value = {"cases": values, "run": run} - elif args.subcommand == "finalize": - value = finalize_run(plan, args.run_id, context_value(args.context)) - elif args.subcommand == "validate": - value = validate_run(plan, args.run_id) - elif args.subcommand == "package": - package_plan(plan, args.run_id, args.output) - value = { - "status": "packaged", - "output": str(args.output), - "sha256": hashlib.sha256(args.output.read_bytes()).hexdigest(), - } - elif args.subcommand == "render": - validate_run(plan, args.run_id) - run, results = render.load_session_results(plan, args.run_id) - rendered = render.render_report(plan, run, results) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(rendered, encoding="utf-8") - value = { - "status": "rendered", - "output": str(args.output), - "bytes": args.output.stat().st_size, - } - else: - raise AssertionError(args.subcommand) - print(json.dumps(value, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == "__main__": - try: - raise SystemExit(main()) - except (DstackTestError, render.ReportError) as error: - print( - json.dumps({"status": "error", "message": str(error)}, ensure_ascii=False), - file=sys.stderr, - ) - raise SystemExit(2) diff --git a/tools/dstack-test/tests/test_dstack_test.py b/tools/dstack-test/tests/test_dstack_test.py index e25712e64..a49d50ae2 100644 --- a/tools/dstack-test/tests/test_dstack_test.py +++ b/tools/dstack-test/tests/test_dstack_test.py @@ -4,6 +4,7 @@ from __future__ import annotations +import importlib.util import json import os import shutil @@ -12,6 +13,7 @@ import tempfile import unittest import zipfile +from importlib.machinery import SourceFileLoader from pathlib import Path HERE = Path(__file__).resolve().parent @@ -20,9 +22,16 @@ CLI = TOOL_DIR / "dstack-test" sys.path.insert(0, str(TOOL_DIR)) -import dstack_test # noqa: E402 import render # noqa: E402 +spec = importlib.util.spec_from_loader( + "dstack_test", SourceFileLoader("dstack_test", str(CLI)) +) +if spec is None or spec.loader is None: + raise RuntimeError(f"failed to load {CLI}") +dstack_test = importlib.util.module_from_spec(spec) +spec.loader.exec_module(dstack_test) + class DstackTestTests(unittest.TestCase): def copy_fixture(self, root: Path) -> Path: @@ -30,6 +39,13 @@ def copy_fixture(self, root: Path) -> Path: shutil.copytree(FIXTURE, plan) return plan + def test_run_command_defaults(self) -> None: + args = dstack_test.build_parser().parse_args( + ["run-plan", "--plan", str(FIXTURE)] + ) + self.assertEqual(args.agent, "codex") + self.assertRegex(args.run_id, r"^run-\d{8}T\d{6}Z-[0-9a-f]{6}$") + def test_validate_and_render_fixture(self) -> None: plan = render.load_plan(FIXTURE) valid = dstack_test.validate_run(plan, "run-demo") From 54e9e06aabf828c0d7ff3b4acbe0ae7846a4ad87 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 23 Jul 2026 03:04:49 -0700 Subject: [PATCH 4/8] fix(testing): align plan example and show run progress --- docs/testing/test-case-authoring-spec.md | 14 +++++++-- tools/dstack-test/dstack-test | 37 +++++++++++++++--------- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/docs/testing/test-case-authoring-spec.md b/docs/testing/test-case-authoring-spec.md index 56330e759..5b4d8900d 100644 --- a/docs/testing/test-case-authoring-spec.md +++ b/docs/testing/test-case-authoring-spec.md @@ -70,23 +70,33 @@ The index is authoritative for discovery and order. A minimal example is: "schema_version": "1.0", "id": "dstack-v0-6-0-release", "title": "dstack v0.6.0 Release Test Plan", - "guide": "README.md", + "guide": {"path": "README.md", "anchor": "release-test-guide"}, "chapters": [ { "id": "chapter-gateway", "title": "Gateway", "order": 1, + "path": "01-gateway", "sections": [ { "id": "section-gateway-proxy-protocol", "title": "Proxy Protocol", "order": 1, + "path": "01-gateway/01-proxy-protocol", "cases": [ { "id": "tc-gw-pp-001", "title": "Forward a Proxy v1 client address over TLS termination", "order": 1, - "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001" + "priority": "P0", + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001", + "spec": { + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md", + "anchor": "tc-gw-pp-001" + }, + "requirements": ["req-gw-pp-001"], + "risks": ["risk-gw-spoofing-001"], + "tags": ["gateway", "proxy-protocol"] } ] } diff --git a/tools/dstack-test/dstack-test b/tools/dstack-test/dstack-test index 0c093118d..ec034a773 100755 --- a/tools/dstack-test/dstack-test +++ b/tools/dstack-test/dstack-test @@ -591,23 +591,34 @@ def main(argv: list[str] | None = None) -> int: ) elif args.subcommand == "run-plan": values = [] - for case in plan.cases: + total = len(plan.cases) + for index, case in enumerate(plan.cases, 1): + print(f"[{index}/{total}] starting {case.id}", file=sys.stderr, flush=True) try: - values.append( - run_case( - plan, - case, - args.run_id, - args.agent, - args.model, - args.workdir.resolve(), - prompt_text(args), - args.agent_arg, - args.overwrite, - ) + case_value = run_case( + plan, + case, + args.run_id, + args.agent, + args.model, + args.workdir.resolve(), + prompt_text(args), + args.agent_arg, + args.overwrite, + ) + values.append(case_value) + print( + f"[{index}/{total}] finished {case.id}: {case_value['status']}", + file=sys.stderr, + flush=True, ) except DstackTestError as error: values.append({"case": case.id, "runner_error": str(error)}) + print( + f"[{index}/{total}] runner error for {case.id}: {error}", + file=sys.stderr, + flush=True, + ) run = finalize_run(plan, args.run_id, context_value(args.context)) value = {"cases": values, "run": run} elif args.subcommand == "finalize": From ea7e3e00cc5bacae47b3832f209c35c8dabf8dcf Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 23 Jul 2026 03:05:57 -0700 Subject: [PATCH 5/8] fix(testing): record configured Codex model --- tools/dstack-test/dstack-test | 23 +++++++++++++++++++-- tools/dstack-test/tests/test_dstack_test.py | 10 +++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/tools/dstack-test/dstack-test b/tools/dstack-test/dstack-test index ec034a773..512f51d6a 100755 --- a/tools/dstack-test/dstack-test +++ b/tools/dstack-test/dstack-test @@ -24,6 +24,7 @@ from pathlib import Path from typing import Any import render +import tomllib TERMINAL_STATUS = ("PASS", "FAIL", "BLOCKED", "NOT_RUN", "SKIPPED") @@ -223,6 +224,20 @@ def agent_command( raise DstackTestError(f"unsupported agent: {agent}") +def resolve_model(agent: str, model: str | None) -> str | None: + if model or agent != "codex": + return model + config_home = Path(os.environ.get("CODEX_HOME", Path.home() / ".codex")) + config_path = config_home / "config.toml" + if not config_path.is_file(): + return None + try: + configured = tomllib.loads(config_path.read_text(encoding="utf-8")).get("model") + except (OSError, tomllib.TOMLDecodeError): + return None + return configured if isinstance(configured, str) and configured else None + + def run_case( plan: render.Plan, case: render.CaseEntry, @@ -248,7 +263,8 @@ def run_case( stderr_path = result_dir / "agent-stderr.log" started_at = utc_now() start = time.monotonic_ns() - command = agent_command(agent, model, workdir, full_prompt, extra) + effective_model = resolve_model(agent, model) + command = agent_command(agent, effective_model, workdir, full_prompt, extra) binary = shutil.which(command[0]) if not binary: raise DstackTestError(f"agent CLI not found: {command[0]}") @@ -272,7 +288,10 @@ def run_case( "schema_version": "1.0", "run_id": run_id, "case_id": case.id, - "agent": {"type": agent, "model": detected_model or model or "unknown"}, + "agent": { + "type": agent, + "model": detected_model or effective_model or "unknown", + }, "session": { "format": f"{agent}-jsonl", "path": "session.jsonl", diff --git a/tools/dstack-test/tests/test_dstack_test.py b/tools/dstack-test/tests/test_dstack_test.py index a49d50ae2..6751df16f 100644 --- a/tools/dstack-test/tests/test_dstack_test.py +++ b/tools/dstack-test/tests/test_dstack_test.py @@ -15,6 +15,7 @@ import zipfile from importlib.machinery import SourceFileLoader from pathlib import Path +from unittest import mock HERE = Path(__file__).resolve().parent TOOL_DIR = HERE.parent @@ -46,6 +47,15 @@ def test_run_command_defaults(self) -> None: self.assertEqual(args.agent, "codex") self.assertRegex(args.run_id, r"^run-\d{8}T\d{6}Z-[0-9a-f]{6}$") + def test_codex_model_is_read_from_config(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + config = Path(temporary) / "config.toml" + config.write_text('model = "test-codex-model"\n', encoding="utf-8") + with mock.patch.dict(os.environ, {"CODEX_HOME": temporary}): + self.assertEqual( + dstack_test.resolve_model("codex", None), "test-codex-model" + ) + def test_validate_and_render_fixture(self) -> None: plan = render.load_plan(FIXTURE) valid = dstack_test.validate_run(plan, "run-demo") From ee240584ad45a9eef7b835e92255dc09f471e139 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 23 Jul 2026 03:22:23 -0700 Subject: [PATCH 6/8] feat(testing): make run-plan dependency aware --- docs/testing/dstack-test-methodology.md | 10 +- docs/testing/test-report-output-spec.md | 8 +- tools/dstack-test/README.md | 12 +- tools/dstack-test/dstack-test | 208 ++++++++++++++++++-- tools/dstack-test/tests/test_dstack_test.py | 14 ++ 5 files changed, 227 insertions(+), 25 deletions(-) diff --git a/docs/testing/dstack-test-methodology.md b/docs/testing/dstack-test-methodology.md index ec95ab8fa..788b89dfc 100644 --- a/docs/testing/dstack-test-methodology.md +++ b/docs/testing/dstack-test-methodology.md @@ -82,7 +82,15 @@ Review the plan for change coverage, regression breadth, compatibility matrices, ### 3.5 Execute -An AI executor must: +The `run-plan` orchestration agent must first read the guide, index, and every +case specification. It processes cases in index order, starts an independent +case-agent session for each runnable case, and reads the completed result before +deciding about later cases. It may mark a later case `SKIPPED` without launching +it only when a recorded earlier non-PASS result demonstrably makes the later +case's prerequisite false or its result meaningless. Similarity, expected cost, +or a mere possibility of failure is not sufficient. Independent cases continue. + +Each case executor must: 1. read the plan `README.md` and `index.json`; 2. execute cases in index order unless the guide explicitly permits parallelism; diff --git a/docs/testing/test-report-output-spec.md b/docs/testing/test-report-output-spec.md index e681e407f..0a4bf72da 100644 --- a/docs/testing/test-report-output-spec.md +++ b/docs/testing/test-report-output-spec.md @@ -8,7 +8,8 @@ This document defines the normative format for AI sessions, case summaries, run ## 1. Principles -1. Each case runs in an independent Codex or Claude session. +1. `run-plan` uses an AI scheduling session before each case, and each case it + executes runs in an independent Codex or Claude session. 2. The agent's native JSONL is the primary source for commands, tool calls, and raw output. 3. The agent writes one shallow `result.json`; it does not copy command output into that file. 4. The runner generates `runner.json`, timestamps, exit code, checksums, and run aggregates. @@ -48,6 +49,11 @@ dstack-test run-case \ The runner supplies the plan guide, `case.md`, output location, status rules, and result schema in the prompt. The agent must read the guide before the case and must not modify plan specifications. +For a dependency-driven skip, the orchestrator creates a synthetic one-event +case session containing the reason and causal earlier case IDs. It does not +pretend that the skipped case was executed. The full decision process remains +available under the run-level `decisions/` directory. + ## 4. Result layout diff --git a/tools/dstack-test/README.md b/tools/dstack-test/README.md index bd3724073..daa1bd9fe 100644 --- a/tools/dstack-test/README.md +++ b/tools/dstack-test/README.md @@ -42,8 +42,16 @@ tools/dstack-test/dstack-test run-plan \ -- "Follow the environment restrictions in README.md" ``` -Every case receives an independent agent session. `run-plan` continues after an -agent/runner failure and finalizes an `INCOMPLETE` run when necessary. +Before each case, `run-plan` starts a scheduling-agent session that reads the +guide, index, all case specifications, and prior results. The runner—not the +scheduling agent—then invokes `run-case` in a fresh session when the decision is +`RUN`. When a +prior non-PASS result demonstrably invalidates a later prerequisite, it records +that later case as `SKIPPED` with the causal case IDs instead of starting an +unnecessary case session. It must not skip independent or merely expensive +cases. The complete orchestration session is stored at +`/results//decisions/`. Keeping scheduling and case execution in +separate processes avoids nested agent CLIs and preserves trust boundaries. ## Finalize and validate diff --git a/tools/dstack-test/dstack-test b/tools/dstack-test/dstack-test index 512f51d6a..cedf3a115 100755 --- a/tools/dstack-test/dstack-test +++ b/tools/dstack-test/dstack-test @@ -319,6 +319,100 @@ def run_case( return {"case": case.id, "status": result["status"], "result_dir": str(result_dir)} +def skip_case( + case: render.CaseEntry, + run_id: str, + reason: str, + blocked_by: list[str], + model: str | None = None, +) -> dict[str, Any]: + result_dir = case.path / "results" / run_id + if result_dir.exists() and any(result_dir.iterdir()): + raise DstackTestError(f"result directory is not empty: {result_dir}") + result_dir.mkdir(parents=True, exist_ok=True) + now = utc_now() + event = { + "type": "orchestrator.skip", + "case_id": case.id, + "reason": reason, + "blocked_by": blocked_by, + } + (result_dir / "session.jsonl").write_text( + json.dumps(event, ensure_ascii=False) + "\n", encoding="utf-8" + ) + result = { + "schema_version": "1.0", + "case_id": case.id, + "status": "SKIPPED", + "summary": reason, + "steps": [], + "artifacts": [], + "remarks": "Skipped by the run-plan orchestrator after evaluating prior results.", + } + atomic_json(result_dir / "result.json", result) + atomic_json( + result_dir / "runner.json", + { + "schema_version": "1.0", + "run_id": run_id, + "case_id": case.id, + "agent": { + "type": "run-plan-scheduler", + "model": model or "unknown", + }, + "session": { + "format": "orchestrator-decision-jsonl", + "path": "session.jsonl", + "events": 1, + }, + "result_path": "result.json", + "started_at": now, + "finished_at": now, + "duration_ms": 0, + "exit_code": 0, + "result_valid": True, + "result_error": None, + }, + ) + write_sha256s(result_dir) + return {"case": case.id, "status": "SKIPPED", "result_dir": str(result_dir)} + + +def orchestration_prompt( + plan: render.Plan, + case: render.CaseEntry, + prior: list[dict[str, Any]], + decision_path: Path, + extra_prompt: str, +) -> str: + prior_json = json.dumps(prior, ensure_ascii=False, indent=2) + return f"""You are the dstack test-plan scheduling agent. Decide whether the next case should be executed; do not execute the case yourself. + +Read the plan guide at {plan.guide_path}, index at {plan.root / "index.json"}, every case specification, and the prior results below. The next case in authoritative order is {case.id} at {case.spec_path}. + +Prior completed results: +{prior_json} + +Decision rules: +- Choose RUN unless a recorded prior non-PASS result demonstrably makes this case's prerequisite false or makes its result meaningless. +- Choose SKIP only for such a proven dependency consequence. Similarity, cost, likely failure, or an independently checkable prerequisite is not sufficient. +- A SKIP decision must name one or more causal earlier case IDs in blocked_by and explain the exact dependency. +- Never reinterpret a product failure as a scheduling failure. Independent cases continue after FAIL, BLOCKED, or SKIPPED. +- Do not modify the plan or any case result. + +Atomically write exactly this shallow JSON object to {decision_path}: +{{ + "decision": "RUN|SKIP", + "reason": "concise evidence-based reason", + "blocked_by": ["prior-case-id"] +}} +For RUN, blocked_by must be empty. Finish after writing the decision. + +Additional caller constraints: +{extra_prompt or "None"} +""" + + def write_sha256s(root: Path) -> None: entries: list[str] = [] for path in sorted( @@ -609,37 +703,109 @@ def main(argv: list[str] | None = None) -> int: args.overwrite, ) elif args.subcommand == "run-plan": - values = [] - total = len(plan.cases) - for index, case in enumerate(plan.cases, 1): - print(f"[{index}/{total}] starting {case.id}", file=sys.stderr, flush=True) - try: - case_value = run_case( + run_dir = plan.root / "results" / args.run_id + existing = run_dir.exists() or any( + (case.path / "results" / args.run_id).exists() for case in plan.cases + ) + if existing and not args.overwrite: + raise DstackTestError( + f"run {args.run_id!r} already exists; pass --overwrite to replace it" + ) + if args.overwrite: + shutil.rmtree(run_dir, ignore_errors=True) + for case in plan.cases: + shutil.rmtree(case.path / "results" / args.run_id, ignore_errors=True) + decisions_dir = run_dir / "decisions" + decisions_dir.mkdir(parents=True, exist_ok=True) + effective_model = resolve_model(args.agent, args.model) + prior: list[dict[str, Any]] = [] + for number, case in enumerate(plan.cases, 1): + print( + f"[{number}/{len(plan.cases)}] scheduling {case.id}", + file=sys.stderr, + flush=True, + ) + decision_path = decisions_dir / f"{case.id}.json" + prompt = orchestration_prompt( + plan, case, prior, decision_path, prompt_text(args) + ) + (decisions_dir / f"{case.id}.prompt.md").write_text( + prompt, encoding="utf-8" + ) + command = agent_command( + args.agent, + effective_model, + args.workdir.resolve(), + prompt, + args.agent_arg, + ) + with ( + (decisions_dir / f"{case.id}.session.jsonl").open("wb") as stdout, + (decisions_dir / f"{case.id}.stderr.log").open("wb") as stderr, + ): + completed = subprocess.run( + command, cwd=args.workdir, stdout=stdout, stderr=stderr, check=False + ) + validate_session(decisions_dir / f"{case.id}.session.jsonl") + if completed.returncode != 0: + raise DstackTestError( + f"scheduler failed for {case.id} with exit code {completed.returncode}" + ) + decision = render.load_json(decision_path) + action = decision.get("decision") + reason = decision.get("reason") + blocked_by = decision.get("blocked_by", []) + if ( + action not in ("RUN", "SKIP") + or not isinstance(reason, str) + or not reason.strip() + or not isinstance(blocked_by, list) + ): + raise DstackTestError(f"invalid scheduling decision for {case.id}") + prior_by_id = {item["case_id"]: item for item in prior} + if action == "SKIP": + if not blocked_by or any( + ref not in prior_by_id or prior_by_id[ref]["status"] == "PASS" + for ref in blocked_by + ): + raise DstackTestError( + f"invalid blocked_by references for {case.id}" + ) + skip_case(case, args.run_id, reason, blocked_by, effective_model) + result = {"case_id": case.id, "status": "SKIPPED", "summary": reason} + else: + if blocked_by: + raise DstackTestError( + f"RUN decision has blocked_by entries for {case.id}" + ) + run_case( plan, case, args.run_id, args.agent, - args.model, + effective_model, args.workdir.resolve(), prompt_text(args), args.agent_arg, - args.overwrite, - ) - values.append(case_value) - print( - f"[{index}/{total}] finished {case.id}: {case_value['status']}", - file=sys.stderr, - flush=True, + False, ) - except DstackTestError as error: - values.append({"case": case.id, "runner_error": str(error)}) - print( - f"[{index}/{total}] runner error for {case.id}: {error}", - file=sys.stderr, - flush=True, + result = validate_summary( + case, case.path / "results" / args.run_id / "result.json" ) + prior.append( + { + "case_id": case.id, + "status": result["status"], + "summary": result["summary"], + } + ) + print( + f"[{number}/{len(plan.cases)}] {case.id}: {result['status']}", + file=sys.stderr, + flush=True, + ) run = finalize_run(plan, args.run_id, context_value(args.context)) - value = {"cases": values, "run": run} + value = {"decisions": prior, "run": run} elif args.subcommand == "finalize": value = finalize_run(plan, args.run_id, context_value(args.context)) elif args.subcommand == "validate": diff --git a/tools/dstack-test/tests/test_dstack_test.py b/tools/dstack-test/tests/test_dstack_test.py index 6751df16f..5a3223f1d 100644 --- a/tools/dstack-test/tests/test_dstack_test.py +++ b/tools/dstack-test/tests/test_dstack_test.py @@ -56,6 +56,20 @@ def test_codex_model_is_read_from_config(self) -> None: dstack_test.resolve_model("codex", None), "test-codex-model" ) + def test_orchestrator_can_record_dependency_skip(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + plan_path = self.copy_fixture(Path(temporary)) + plan = render.load_plan(plan_path) + case = plan.cases[0] + value = dstack_test.skip_case( + case, "run-skip", "prerequisite case failed", ["tc-prereq-001"] + ) + self.assertEqual(value["status"], "SKIPPED") + result_dir = case.path / "results" / "run-skip" + result = dstack_test.validate_summary(case, result_dir / "result.json") + self.assertEqual(result["status"], "SKIPPED") + self.assertIn("tc-prereq-001", (result_dir / "session.jsonl").read_text()) + def test_validate_and_render_fixture(self) -> None: plan = render.load_plan(FIXTURE) valid = dstack_test.validate_run(plan, "run-demo") From 770eb2f25a0b80be30510b01b60600031e4f0870 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 23 Jul 2026 03:45:23 -0700 Subject: [PATCH 7/8] feat(testing): let AI orchestrate the plan loop --- docs/testing/test-report-output-spec.md | 6 +- tools/dstack-test/README.md | 19 +- tools/dstack-test/dstack-test | 374 ++++++++++++++++-------- 3 files changed, 269 insertions(+), 130 deletions(-) diff --git a/docs/testing/test-report-output-spec.md b/docs/testing/test-report-output-spec.md index 0a4bf72da..50391d4f8 100644 --- a/docs/testing/test-report-output-spec.md +++ b/docs/testing/test-report-output-spec.md @@ -8,8 +8,8 @@ This document defines the normative format for AI sessions, case summaries, run ## 1. Principles -1. `run-plan` uses an AI scheduling session before each case, and each case it - executes runs in an independent Codex or Claude session. +1. `run-plan` uses one AI orchestration session to drive the `next-case` loop, + and each case it executes runs in an independent Codex or Claude session. 2. The agent's native JSONL is the primary source for commands, tool calls, and raw output. 3. The agent writes one shallow `result.json`; it does not copy command output into that file. 4. The runner generates `runner.json`, timestamps, exit code, checksums, and run aggregates. @@ -52,7 +52,7 @@ The runner supplies the plan guide, `case.md`, output location, status rules, an For a dependency-driven skip, the orchestrator creates a synthetic one-event case session containing the reason and causal earlier case IDs. It does not pretend that the skipped case was executed. The full decision process remains -available under the run-level `decisions/` directory. +available in the run-level `orchestrator.jsonl`. ## 4. Result layout diff --git a/tools/dstack-test/README.md b/tools/dstack-test/README.md index daa1bd9fe..bd2cb7894 100644 --- a/tools/dstack-test/README.md +++ b/tools/dstack-test/README.md @@ -42,16 +42,15 @@ tools/dstack-test/dstack-test run-plan \ -- "Follow the environment restrictions in README.md" ``` -Before each case, `run-plan` starts a scheduling-agent session that reads the -guide, index, all case specifications, and prior results. The runner—not the -scheduling agent—then invokes `run-case` in a fresh session when the decision is -`RUN`. When a -prior non-PASS result demonstrably invalidates a later prerequisite, it records -that later case as `SKIPPED` with the causal case IDs instead of starting an -unnecessary case session. It must not skip independent or merely expensive -cases. The complete orchestration session is stored at -`/results//decisions/`. Keeping scheduling and case execution in -separate processes avoids nested agent CLIs and preserves trust boundaries. +`run-plan` starts one orchestration-agent session. The agent drives the loop by +calling `next-case`, followed by either `run-case` or `complete-case`, until the +server reports `COMPLETE`. A file-based control channel lets the outer runner +launch each case agent outside the orchestrator sandbox. When a prior non-PASS +result demonstrably invalidates a later prerequisite, the orchestrator records +that case as `SKIPPED`; it must not skip independent or merely expensive cases. +The complete control conversation is stored in `orchestrator.jsonl`. + +An interrupted run can be continued with the same run ID and `--resume`. ## Finalize and validate diff --git a/tools/dstack-test/dstack-test b/tools/dstack-test/dstack-test index cedf3a115..65baade11 100755 --- a/tools/dstack-test/dstack-test +++ b/tools/dstack-test/dstack-test @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: © 2026 Phala Network # SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: D100, D101, D102, D103 +# ruff: noqa: D100, D101, D102, D103, D107 """Run, validate, package, and render dstack test plans.""" from __future__ import annotations @@ -17,6 +17,7 @@ import subprocess import sys import tarfile import tempfile +import threading import time import zipfile from datetime import UTC, datetime @@ -378,41 +379,187 @@ def skip_case( return {"case": case.id, "status": "SKIPPED", "result_dir": str(result_dir)} -def orchestration_prompt( - plan: render.Plan, - case: render.CaseEntry, - prior: list[dict[str, Any]], - decision_path: Path, - extra_prompt: str, -) -> str: - prior_json = json.dumps(prior, ensure_ascii=False, indent=2) - return f"""You are the dstack test-plan scheduling agent. Decide whether the next case should be executed; do not execute the case yourself. +def orchestration_prompt(plan: render.Plan, extra_prompt: str) -> str: + return f"""You are the dstack test-plan orchestrator. You drive the entire loop. -Read the plan guide at {plan.guide_path}, index at {plan.root / "index.json"}, every case specification, and the prior results below. The next case in authoritative order is {case.id} at {case.spec_path}. +Read {plan.guide_path}, {plan.root / "index.json"}, and all indexed case specifications first. Then repeatedly call `{Path(__file__).resolve()} next-case`. It returns the first unresolved case, its description, and prior results. For each READY case, choose exactly one action: -Prior completed results: -{prior_json} +- Run it with `{Path(__file__).resolve()} run-case --case `. +- Only when a prior non-PASS result demonstrably invalidates an explicit prerequisite, record it with `{Path(__file__).resolve()} complete-case --case --result SKIPPED --blocked-by --reason `. -Decision rules: -- Choose RUN unless a recorded prior non-PASS result demonstrably makes this case's prerequisite false or makes its result meaningless. -- Choose SKIP only for such a proven dependency consequence. Similarity, cost, likely failure, or an independently checkable prerequisite is not sufficient. -- A SKIP decision must name one or more causal earlier case IDs in blocked_by and explain the exact dependency. -- Never reinterpret a product failure as a scheduling failure. Independent cases continue after FAIL, BLOCKED, or SKIPPED. -- Do not modify the plan or any case result. - -Atomically write exactly this shallow JSON object to {decision_path}: -{{ - "decision": "RUN|SKIP", - "reason": "concise evidence-based reason", - "blocked_by": ["prior-case-id"] -}} -For RUN, blocked_by must be empty. Finish after writing the decision. +After either action, call next-case again. You may issue the action and next-case as two commands in one shell tool call. Continue until next-case returns COMPLETE. Do not skip for cost, similarity, likely failure, or independently checkable prerequisites. Do not edit plan or result files yourself. Do not call run-plan, finalize, validate, package, or render. Additional caller constraints: {extra_prompt or "None"} """ +class ControlState: + def __init__(self, plan: render.Plan, args: argparse.Namespace, model: str | None): + self.plan, self.args, self.model = plan, args, model + self.lock = threading.Lock() + + def prior(self) -> list[dict[str, Any]]: + values = [] + for case in self.plan.cases: + path = case.path / "results" / self.args.run_id / "result.json" + if path.is_file(): + result = validate_summary(case, path) + values.append( + { + "case_id": case.id, + "status": result["status"], + "summary": result["summary"], + } + ) + return values + + def next_case(self, *, announce: bool = True) -> dict[str, Any]: + prior = self.prior() + done = {item["case_id"] for item in prior} + for number, case in enumerate(self.plan.cases, 1): + if case.id not in done: + if announce: + print( + f"[{number}/{len(self.plan.cases)}] next case: {case.id}", + file=sys.stderr, + flush=True, + ) + return { + "status": "READY", + "position": {"index": number, "total": len(self.plan.cases)}, + "case": { + "id": case.id, + "title": case.title, + "spec_path": str(case.spec_path), + "specification": case.spec_path.read_text(encoding="utf-8"), + }, + "prior_results": prior, + } + return { + "status": "COMPLETE", + "completed": len(done), + "total": len(self.plan.cases), + } + + def current(self, case_id: str) -> tuple[render.CaseEntry, int]: + next_value = self.next_case(announce=False) + if next_value["status"] != "READY" or next_value["case"]["id"] != case_id: + raise DstackTestError(f"{case_id} is not the current unresolved case") + return find_case(self.plan, case_id), next_value["position"]["index"] + + def handle(self, request: dict[str, Any]) -> dict[str, Any]: + with self.lock: + action = request.get("action") + if action == "next-case": + return self.next_case() + case, number = self.current(str(request.get("case"))) + if action == "run-case": + print( + f"[{number}/{len(self.plan.cases)}] executing {case.id}", + file=sys.stderr, + flush=True, + ) + value = run_case( + self.plan, + case, + self.args.run_id, + self.args.agent, + self.model, + self.args.workdir.resolve(), + prompt_text(self.args), + self.args.agent_arg, + False, + ) + print( + f"[{number}/{len(self.plan.cases)}] finished {case.id}: {value['status']}", + file=sys.stderr, + flush=True, + ) + return value + if action == "complete-case": + if request.get("result") != "SKIPPED": + raise DstackTestError( + "complete-case currently accepts only SKIPPED" + ) + blocked_by = request.get("blocked_by", []) + prior = {item["case_id"]: item for item in self.prior()} + if not blocked_by or any( + ref not in prior or prior[ref]["status"] == "PASS" + for ref in blocked_by + ): + raise DstackTestError( + "blocked_by must reference prior non-PASS cases" + ) + value = skip_case( + case, + self.args.run_id, + str(request.get("reason", "")), + blocked_by, + self.model, + ) + print( + f"[{number}/{len(self.plan.cases)}] skipped {case.id}", + file=sys.stderr, + flush=True, + ) + return value + raise DstackTestError(f"unsupported control action: {action}") + + +def control_request(request: dict[str, Any]) -> dict[str, Any]: + control_dir = os.environ.get("DSTACK_TEST_CONTROL_DIR") + if not control_dir: + raise DstackTestError("command requires an active run-plan control server") + root = Path(control_dir) + request_id = secrets.token_hex(12) + request_path = root / f"{request_id}.request.json" + response_path = root / f"{request_id}.response.json" + atomic_json(request_path, request) + deadline = time.monotonic() + 86400 + while not response_path.is_file(): + if time.monotonic() > deadline: + raise DstackTestError("control request timed out") + time.sleep(0.1) + response = render.load_json(response_path) + request_path.unlink(missing_ok=True) + response_path.unlink(missing_ok=True) + if not response.get("ok"): + raise DstackTestError(response.get("error", "control request failed")) + return response["value"] + + +class FileControlServer: + def __init__(self, state: ControlState, root: Path): + self.state, self.root = state, root + self.stop_event = threading.Event() + self.thread = threading.Thread(target=self.serve, daemon=True) + + def start(self) -> None: + self.root.mkdir(parents=True, exist_ok=True) + self.thread.start() + + def serve(self) -> None: + while not self.stop_event.is_set(): + for request_path in sorted(self.root.glob("*.request.json")): + response_path = request_path.with_name( + request_path.name.replace(".request.json", ".response.json") + ) + if response_path.exists(): + continue + try: + value = self.state.handle(render.load_json(request_path)) + response = {"ok": True, "value": value} + except Exception as error: + response = {"ok": False, "error": str(error)} + atomic_json(response_path, response) + self.stop_event.wait(0.05) + + def close(self) -> None: + self.stop_event.set() + self.thread.join() + + def write_sha256s(root: Path) -> None: entries: list[str] = [] for path in sorted( @@ -606,9 +753,14 @@ def package_plan(plan: render.Plan, run_id: str, output: Path) -> None: def add_common_plan( - parser: argparse.ArgumentParser, *, run_id_required: bool = True + parser: argparse.ArgumentParser, + *, + run_id_required: bool = True, + plan_required: bool = True, ) -> None: - parser.add_argument("--plan", type=Path, required=True, help="test-plan directory") + parser.add_argument( + "--plan", type=Path, required=plan_required, help="test-plan directory" + ) parser.add_argument( "--run-id", required=run_id_required, @@ -624,7 +776,7 @@ def build_parser() -> argparse.ArgumentParser: run_case_parser = sub.add_parser( "run-case", help="run one indexed case in a fresh AI session" ) - add_common_plan(run_case_parser, run_id_required=False) + add_common_plan(run_case_parser, run_id_required=False, plan_required=False) run_case_parser.add_argument( "--case", required=True, help="case ID or indexed case directory" ) @@ -653,8 +805,18 @@ def build_parser() -> argparse.ArgumentParser: ) run_plan.add_argument("--agent-arg", action="append", default=[]) run_plan.add_argument("--overwrite", action="store_true") + run_plan.add_argument("--resume", action="store_true") run_plan.add_argument("prompt", nargs=argparse.REMAINDER) + sub.add_parser("next-case", help="return the next unresolved controlled case") + complete = sub.add_parser( + "complete-case", help="complete a controlled case without execution" + ) + complete.add_argument("--case", required=True) + complete.add_argument("--result", choices=("SKIPPED",), required=True) + complete.add_argument("--blocked-by", action="append", default=[]) + complete.add_argument("--reason", required=True) + finalize = sub.add_parser("finalize", help="materialize run.json from case results") add_common_plan(finalize) finalize.add_argument("--context", type=Path) @@ -689,6 +851,31 @@ def prompt_text(args: argparse.Namespace) -> str: def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) + if os.environ.get("DSTACK_TEST_CONTROL_DIR") and args.subcommand in ( + "next-case", + "run-case", + "complete-case", + ): + if args.subcommand == "next-case": + value = control_request({"action": "next-case"}) + elif args.subcommand == "run-case": + value = control_request({"action": "run-case", "case": args.case}) + else: + value = control_request( + { + "action": "complete-case", + "case": args.case, + "result": args.result, + "blocked_by": args.blocked_by, + "reason": args.reason, + } + ) + print(json.dumps(value, ensure_ascii=False, indent=2)) + return 0 + if not getattr(args, "plan", None): + raise DstackTestError( + "--plan is required outside a controlled run-plan session" + ) plan = render.load_plan(args.plan) if args.subcommand == "run-case": value = run_case( @@ -707,105 +894,58 @@ def main(argv: list[str] | None = None) -> int: existing = run_dir.exists() or any( (case.path / "results" / args.run_id).exists() for case in plan.cases ) - if existing and not args.overwrite: + if existing and not (args.overwrite or args.resume): raise DstackTestError( - f"run {args.run_id!r} already exists; pass --overwrite to replace it" + f"run {args.run_id!r} already exists; pass --resume or --overwrite" ) if args.overwrite: shutil.rmtree(run_dir, ignore_errors=True) for case in plan.cases: shutil.rmtree(case.path / "results" / args.run_id, ignore_errors=True) - decisions_dir = run_dir / "decisions" - decisions_dir.mkdir(parents=True, exist_ok=True) + run_dir.mkdir(parents=True, exist_ok=True) effective_model = resolve_model(args.agent, args.model) - prior: list[dict[str, Any]] = [] - for number, case in enumerate(plan.cases, 1): - print( - f"[{number}/{len(plan.cases)}] scheduling {case.id}", - file=sys.stderr, - flush=True, - ) - decision_path = decisions_dir / f"{case.id}.json" - prompt = orchestration_prompt( - plan, case, prior, decision_path, prompt_text(args) - ) - (decisions_dir / f"{case.id}.prompt.md").write_text( - prompt, encoding="utf-8" - ) - command = agent_command( - args.agent, - effective_model, - args.workdir.resolve(), - prompt, - args.agent_arg, - ) + state = ControlState(plan, args, effective_model) + control_dir = run_dir / "control" + server = FileControlServer(state, control_dir) + server.start() + prompt = orchestration_prompt(plan, prompt_text(args)) + (run_dir / "orchestrator-prompt.md").write_text(prompt, encoding="utf-8") + environment = os.environ.copy() + environment["DSTACK_TEST_CONTROL_DIR"] = str(control_dir) + command = agent_command( + args.agent, effective_model, args.workdir.resolve(), prompt, args.agent_arg + ) + try: with ( - (decisions_dir / f"{case.id}.session.jsonl").open("wb") as stdout, - (decisions_dir / f"{case.id}.stderr.log").open("wb") as stderr, + (run_dir / "orchestrator.jsonl").open("wb") as stdout, + (run_dir / "orchestrator-stderr.log").open("wb") as stderr, ): completed = subprocess.run( - command, cwd=args.workdir, stdout=stdout, stderr=stderr, check=False - ) - validate_session(decisions_dir / f"{case.id}.session.jsonl") - if completed.returncode != 0: - raise DstackTestError( - f"scheduler failed for {case.id} with exit code {completed.returncode}" - ) - decision = render.load_json(decision_path) - action = decision.get("decision") - reason = decision.get("reason") - blocked_by = decision.get("blocked_by", []) - if ( - action not in ("RUN", "SKIP") - or not isinstance(reason, str) - or not reason.strip() - or not isinstance(blocked_by, list) - ): - raise DstackTestError(f"invalid scheduling decision for {case.id}") - prior_by_id = {item["case_id"]: item for item in prior} - if action == "SKIP": - if not blocked_by or any( - ref not in prior_by_id or prior_by_id[ref]["status"] == "PASS" - for ref in blocked_by - ): - raise DstackTestError( - f"invalid blocked_by references for {case.id}" - ) - skip_case(case, args.run_id, reason, blocked_by, effective_model) - result = {"case_id": case.id, "status": "SKIPPED", "summary": reason} - else: - if blocked_by: - raise DstackTestError( - f"RUN decision has blocked_by entries for {case.id}" - ) - run_case( - plan, - case, - args.run_id, - args.agent, - effective_model, - args.workdir.resolve(), - prompt_text(args), - args.agent_arg, - False, + command, + cwd=args.workdir, + env=environment, + stdout=stdout, + stderr=stderr, + check=False, ) - result = validate_summary( - case, case.path / "results" / args.run_id / "result.json" - ) - prior.append( - { - "case_id": case.id, - "status": result["status"], - "summary": result["summary"], - } - ) - print( - f"[{number}/{len(plan.cases)}] {case.id}: {result['status']}", - file=sys.stderr, - flush=True, - ) + finally: + server.close() + shutil.rmtree(control_dir, ignore_errors=True) + events, detected_model = validate_session(run_dir / "orchestrator.jsonl") + atomic_json( + run_dir / "orchestrator-runner.json", + { + "schema_version": "1.0", + "agent": { + "type": args.agent, + "model": detected_model or effective_model or "unknown", + }, + "events": events, + "exit_code": completed.returncode, + }, + ) run = finalize_run(plan, args.run_id, context_value(args.context)) - value = {"decisions": prior, "run": run} + value = {"orchestrator_exit_code": completed.returncode, "run": run} elif args.subcommand == "finalize": value = finalize_run(plan, args.run_id, context_value(args.context)) elif args.subcommand == "validate": From 14ca28f9a7996cb9b8f2be9628cafff128537f21 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 23 Jul 2026 03:52:18 -0700 Subject: [PATCH 8/8] feat(testing): add live run dashboard --- docs/testing/test-report-output-spec.md | 11 +++ tools/dstack-test/README.md | 15 ++++ tools/dstack-test/dstack-test | 94 +++++++++++++++++++++ tools/dstack-test/tests/test_dstack_test.py | 8 ++ tools/dstack-test/web.py | 82 ++++++++++++++++++ 5 files changed, 210 insertions(+) create mode 100644 tools/dstack-test/web.py diff --git a/docs/testing/test-report-output-spec.md b/docs/testing/test-report-output-spec.md index 50391d4f8..d3efcb284 100644 --- a/docs/testing/test-report-output-spec.md +++ b/docs/testing/test-report-output-spec.md @@ -261,3 +261,14 @@ The output must work offline and inline all CSS, JavaScript, session events, tex - collapsible original agent messages, tool calls, command output, and errors; - every attachment and the complete raw session; and - stable cross-reference anchors. + + +## 13. Live and historical dashboard + +`run-plan --web` starts a read-only HTTP dashboard for case status and the +native JSONL output of the orchestrator and every case agent. The browser polls +incremental byte ranges so active output appears without rewriting session +files. `dstack-test serve --plan --run-id ` exposes the same view +for a completed or interrupted run. The server has no built-in authentication; +non-loopback binding is permitted only on a trusted network or behind an +authenticated tunnel. diff --git a/tools/dstack-test/README.md b/tools/dstack-test/README.md index bd2cb7894..9c8393a5b 100644 --- a/tools/dstack-test/README.md +++ b/tools/dstack-test/README.md @@ -39,6 +39,7 @@ with repeatable `--agent-arg`. The runner writes only below: tools/dstack-test/dstack-test run-plan \ --plan path/to/plan \ --context run-context.json \ + --web \ -- "Follow the environment restrictions in README.md" ``` @@ -52,6 +53,20 @@ The complete control conversation is stored in `orchestrator.jsonl`. An interrupted run can be continued with the same run ID and `--resume`. +`--web` starts a read-only live dashboard on `127.0.0.1` and an automatically +selected port. Use `--web-host 0.0.0.0 --web-port 8000` for remote access; +there is no built-in authentication, so expose it only on a trusted network or +through an authenticated tunnel. The dashboard shows case status and polls the +native orchestrator/case JSONL files for near-real-time output. + +After a run has finished, serve the same historical sessions with: + +```bash +tools/dstack-test/dstack-test serve \ + --plan path/to/plan --run-id run-20260723-001 \ + --host 127.0.0.1 --port 8000 +``` + ## Finalize and validate ```bash diff --git a/tools/dstack-test/dstack-test b/tools/dstack-test/dstack-test index 65baade11..d9a1a6a89 100755 --- a/tools/dstack-test/dstack-test +++ b/tools/dstack-test/dstack-test @@ -26,6 +26,7 @@ from typing import Any import render import tomllib +import web TERMINAL_STATUS = ("PASS", "FAIL", "BLOCKED", "NOT_RUN", "SKIPPED") @@ -560,6 +561,59 @@ class FileControlServer: self.thread.join() +def dashboard_state(plan: render.Plan, run_id: str) -> dict[str, Any]: + run_path = plan.root / "results" / run_id / "run.json" + run = render.load_json(run_path) if run_path.is_file() else {} + counts = {status: 0 for status in (*TERMINAL_STATUS, "RUNNING", "PENDING")} + cases = [] + for case in plan.cases: + result_path = case.path / "results" / run_id / "result.json" + session_path = case.path / "results" / run_id / "session.jsonl" + if result_path.is_file(): + status = str(render.load_json(result_path).get("status", "PENDING")) + else: + status = "RUNNING" if session_path.is_file() else "PENDING" + counts[status] = counts.get(status, 0) + 1 + cases.append({"id": case.id, "title": case.title, "status": status}) + orchestrator = plan.root / "results" / run_id / "orchestrator.jsonl" + return { + "title": plan.index.get("title", plan.index["id"]), + "run_id": run_id, + "run_status": run.get("status", "RUNNING"), + "orchestrator_status": "PASS" + if run_path.is_file() + else ("RUNNING" if orchestrator.is_file() else "PENDING"), + "counts": counts, + "cases": cases, + } + + +def dashboard_log( + plan: render.Plan, run_id: str, agent: str, offset: int +) -> dict[str, Any]: + if agent == "orchestrator": + path = plan.root / "results" / run_id / "orchestrator.jsonl" + elif agent.startswith("case:"): + case = find_case(plan, agent.removeprefix("case:")) + path = case.path / "results" / run_id / "session.jsonl" + else: + raise DstackTestError("unknown agent session") + if not path.is_file(): + return {"text": "", "next_offset": 0, "reset": offset > 0} + size = path.stat().st_size + reset = offset > size + with path.open("rb") as source: + source.seek(0 if reset else offset) + data = source.read(1024 * 1024) + next_offset = source.tell() + return { + "text": data.decode("utf-8", errors="replace"), + "next_offset": next_offset, + "reset": reset, + "eof": next_offset >= size, + } + + def write_sha256s(root: Path) -> None: entries: list[str] = [] for path in sorted( @@ -806,6 +860,9 @@ def build_parser() -> argparse.ArgumentParser: run_plan.add_argument("--agent-arg", action="append", default=[]) run_plan.add_argument("--overwrite", action="store_true") run_plan.add_argument("--resume", action="store_true") + run_plan.add_argument("--web", action="store_true", help="serve a live dashboard") + run_plan.add_argument("--web-host", default="127.0.0.1") + run_plan.add_argument("--web-port", type=int, default=0) run_plan.add_argument("prompt", nargs=argparse.REMAINDER) sub.add_parser("next-case", help="return the next unresolved controlled case") @@ -835,6 +892,11 @@ def build_parser() -> argparse.ArgumentParser: ) add_common_plan(render_parser) render_parser.add_argument("--output", type=Path, required=True) + + serve = sub.add_parser("serve", help="serve a live or historical run dashboard") + add_common_plan(serve) + serve.add_argument("--host", default="127.0.0.1") + serve.add_argument("--port", type=int, default=8000) return parser @@ -908,6 +970,21 @@ def main(argv: list[str] | None = None) -> int: control_dir = run_dir / "control" server = FileControlServer(state, control_dir) server.start() + dashboard = None + if args.web: + dashboard = web.Dashboard( + lambda: dashboard_state(plan, args.run_id), + lambda agent, offset: dashboard_log(plan, args.run_id, agent, offset), + args.web_host, + args.web_port, + ) + dashboard.start() + web_host, web_port = dashboard.address + print( + f"dashboard: http://{web_host}:{web_port}/", + file=sys.stderr, + flush=True, + ) prompt = orchestration_prompt(plan, prompt_text(args)) (run_dir / "orchestrator-prompt.md").write_text(prompt, encoding="utf-8") environment = os.environ.copy() @@ -931,6 +1008,8 @@ def main(argv: list[str] | None = None) -> int: finally: server.close() shutil.rmtree(control_dir, ignore_errors=True) + if dashboard: + dashboard.close() events, detected_model = validate_session(run_dir / "orchestrator.jsonl") atomic_json( run_dir / "orchestrator-runner.json", @@ -968,6 +1047,21 @@ def main(argv: list[str] | None = None) -> int: "output": str(args.output), "bytes": args.output.stat().st_size, } + elif args.subcommand == "serve": + dashboard = web.Dashboard( + lambda: dashboard_state(plan, args.run_id), + lambda agent, offset: dashboard_log(plan, args.run_id, agent, offset), + args.host, + args.port, + ) + dashboard.start() + host, port = dashboard.address + print(f"dashboard: http://{host}:{port}/", file=sys.stderr, flush=True) + try: + dashboard.thread.join() + except KeyboardInterrupt: + dashboard.close() + return 0 else: raise AssertionError(args.subcommand) print(json.dumps(value, ensure_ascii=False, indent=2)) diff --git a/tools/dstack-test/tests/test_dstack_test.py b/tools/dstack-test/tests/test_dstack_test.py index 5a3223f1d..faa2a6c6f 100644 --- a/tools/dstack-test/tests/test_dstack_test.py +++ b/tools/dstack-test/tests/test_dstack_test.py @@ -81,6 +81,14 @@ def test_validate_and_render_fixture(self) -> None: self.assertIn("198.51.100.27:45678", output) self.assertIn("session-tc-gw-pp-001-event-2", output) + def test_dashboard_exposes_historical_status_and_log(self) -> None: + plan = render.load_plan(FIXTURE) + state = dstack_test.dashboard_state(plan, "run-demo") + self.assertEqual(state["cases"][0]["status"], "PASS") + log = dstack_test.dashboard_log(plan, "run-demo", "case:tc-gw-pp-001", 0) + self.assertGreater(log["next_offset"], 0) + self.assertIn("thread.started", log["text"]) + def test_finalize_rebuilds_run_summary(self) -> None: with tempfile.TemporaryDirectory() as temporary: plan_path = self.copy_fixture(Path(temporary)) diff --git a/tools/dstack-test/web.py b/tools/dstack-test/web.py new file mode 100644 index 000000000..a796cfbde --- /dev/null +++ b/tools/dstack-test/web.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +"""Embedded live dashboard for dstack test runs.""" + +import http.server +import json +import threading +import urllib.parse +from typing import Any, Callable + +HTML = r"""dstack test

dstack test

Connecting…
Plan orchestrator
"""
+
+
+class Dashboard:
+    """Serve a read-only live run dashboard."""
+
+    def __init__(
+        self,
+        state: Callable[[], dict[str, Any]],
+        log: Callable[[str, int], dict[str, Any]],
+        host: str,
+        port: int,
+    ):
+        class Handler(http.server.BaseHTTPRequestHandler):
+            def log_message(self, _format: str, *_args: Any) -> None:
+                pass
+
+            def reply(self, value: Any, status: int = 200) -> None:
+                data = json.dumps(value, ensure_ascii=False).encode()
+                self.send_response(status)
+                self.send_header("Content-Type", "application/json; charset=utf-8")
+                self.send_header("Cache-Control", "no-store")
+                self.send_header("Content-Length", str(len(data)))
+                self.end_headers()
+                self.wfile.write(data)
+
+            def do_GET(self) -> None:
+                parsed = urllib.parse.urlparse(self.path)
+                if parsed.path == "/":
+                    data = HTML.encode()
+                    self.send_response(200)
+                    self.send_header("Content-Type", "text/html; charset=utf-8")
+                    self.send_header("Content-Length", str(len(data)))
+                    self.end_headers()
+                    self.wfile.write(data)
+                elif parsed.path == "/api/state":
+                    self.reply(state())
+                elif parsed.path == "/api/log":
+                    query = urllib.parse.parse_qs(parsed.query)
+                    try:
+                        self.reply(
+                            log(
+                                query.get("agent", [""])[0],
+                                int(query.get("offset", ["0"])[0]),
+                            )
+                        )
+                    except Exception as error:  # noqa: BLE001 - API boundary
+                        self.reply({"error": str(error)}, 400)
+                else:
+                    self.send_error(404)
+
+        self.server = http.server.ThreadingHTTPServer((host, port), Handler)
+        self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+
+    @property
+    def address(self) -> tuple[str, int]:
+        host, port = self.server.server_address[:2]
+        return str(host), int(port)
+
+    def start(self) -> None:
+        self.thread.start()
+
+    def close(self) -> None:
+        self.server.shutdown()
+        self.server.server_close()
+        self.thread.join()