diff --git a/REUSE.toml b/REUSE.toml index f68f2a21a..240cc30da 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -203,6 +203,11 @@ path = "**/src/generated/*" SPDX-FileCopyrightText = "NONE" SPDX-License-Identifier = "CC0-1.0" +[[annotations]] +path = "tools/dstack-test/tests/fixtures/**" +SPDX-FileCopyrightText = "NONE" +SPDX-License-Identifier = "CC0-1.0" + [[annotations]] path = "dstack/gateway/assets/*" SPDX-FileCopyrightText = "NONE" diff --git a/docs/testing/dstack-test-methodology.md b/docs/testing/dstack-test-methodology.md new file mode 100644 index 000000000..788b89dfc --- /dev/null +++ b/docs/testing/dstack-test-methodology.md @@ -0,0 +1,145 @@ + + + +# dstack Test Methodology + +This document defines the common process for dstack release testing, from change analysis and risk assessment through execution, evidence collection, and release decisions. See the [test-case authoring specification](test-case-authoring-spec.md#dstack-test-case-authoring-spec) and [report output specification](test-report-output-spec.md#dstack-test-report-output-spec) for normative formats. + + +## 1. Objectives + +Testing must produce reproducible, auditable, and traceable release evidence—not merely show that a script once exited successfully. A conclusion must be traceable from a requirement or risk to a case, step, original command evidence, observation, and attachment. + +Testing is complete only when: + +1. every relevant change, requirement, and material risk has explicit coverage; +2. an executor unfamiliar with the implementation can reproduce each case; +3. the native AI session preserves executed commands and their raw output; +4. simulated and physical-hardware results are reported separately; +5. tools can recompute aggregate status from atomic case results; and +6. references, attachment digests, and statistics are machine-verifiable. + + +## 2. Artifact layers + +Do not mix these four layers: + +| Layer | Purpose | Immutable after execution starts | +|---|---|---:| +| Change audit | Establishes changed behavior, dependencies, and risks | Yes | +| Test plan | Defines scope, topology, cases, and execution order | Yes | +| Case specification | Defines preconditions, actions, and expected results | Yes | +| `results//` | Records versions, native sessions, observations, and attachments | No, while running | + +A plan uses exactly three semantic levels: chapter, section, and case. Its machine-readable execution order is defined by `index.json`; its top-level `README.md` is the executor's environment guide. + + +## 3. Workflow + +### 3.1 Audit the release delta + +Compare the previous released tag with the candidate commit. Inspect commits, pull requests, schemas, RPCs, command-line interfaces, configuration defaults, systemd units, image recipes, deployment manifests, migrations, and dependency changes. For every change record: + +- the user-visible or operational behavior; +- affected components and interfaces; +- compatibility direction and version combinations; +- failure modes and security impact; +- the requirement and risk IDs used by test cases; and +- whether physical TEE hardware is required. + +Generated changelogs alone are insufficient. Follow data and control flow across component boundaries. + +### 3.2 Build a risk-based coverage matrix + +Classify coverage as: + +- **new or changed functionality**: full positive, boundary, and relevant negative coverage; +- **regression**: behavior likely to be affected by shared code, configuration, images, protocols, or lifecycle changes; +- **compatibility**: supported mixed-version combinations and upgrade order; +- **security**: trust boundaries, identity, attestation, key handling, authorization, and secret disclosure; +- **operations**: install, upgrade, restart, recovery, logging, and diagnostics. + +Prioritize by impact, likelihood, detectability, and breadth. `P0` covers release-blocking trust, data-loss, availability, or primary-path risks; `P1` covers important supported behavior; `P2` covers lower-risk variants. + +### 3.3 Define environments + +The plan guide must describe topology, component endpoints, credentials, test data, health checks, concurrency constraints, cleanup, and prohibited operations. Record common software versions once in run-level context. A case records a version override only when it deliberately uses a different component version. + +Environment levels are: + +- **UNIT**: isolated code-level validation; +- **SIMULATOR**: no-TEE or mock-attestation execution; +- **INTEGRATION**: deployed multi-component system; +- **HARDWARE**: physical supported TEE hardware. + +Simulation may follow `docs/development-without-tee.md`; a local key provider that cannot start without TPM support may use TPM mode. Simulation never proves hardware-specific boot, measurement, attestation, sealing, or device behavior. Such unconfirmed items must be called out separately in the report. + +### 3.4 Author and review cases + +Each case validates one independently decidable behavior and references at least one requirement or risk. Prefer three to eight logical steps. Every step defines an action and exact observable expected results. Do not write a separate failure criterion: any result that does not fully match the expected result is `FAIL`. + +Review the plan for change coverage, regression breadth, compatibility matrices, security boundaries, operational recovery, test-data isolation, and cleanup before execution. + +### 3.5 Execute + +The `run-plan` orchestration agent must first read the guide, index, and every +case specification. It processes cases in index order, starts an independent +case-agent session for each runnable case, and reads the completed result before +deciding about later cases. It may mark a later case `SKIPPED` without launching +it only when a recorded earlier non-PASS result demonstrably makes the later +case's prerequisite false or its result meaningless. Similarity, expected cost, +or a mere possibility of failure is not sufficient. Independent cases continue. + +Each case executor must: + +1. read the plan `README.md` and `index.json`; +2. execute cases in index order unless the guide explicitly permits parallelism; +3. start a fresh Codex or Claude session for each case; +4. execute real commands rather than infer outcomes; +5. preserve the native JSONL session as step evidence; +6. write only a shallow atomic `result.json`; and +7. continue to later independent cases after a case-level failure. + +The executor name and model are recorded by the runner. Secrets must never be emitted into sessions or artifacts. + + +## 4. Status model + +Case and step status is one of: + +- `PASS`: every expected result was fully observed; +- `FAIL`: at least one expected result was not fully observed; +- `BLOCKED`: an external prerequisite prevented the tested behavior from starting; +- `NOT_RUN`: execution was not attempted; +- `SKIPPED`: omission was explicitly authorized and explained. + +`PARTIAL` is forbidden. A completed run may contain any terminal case status. A run is `INCOMPLETE` only when required case result artifacts are missing. + +Product failure and test-infrastructure failure must be distinguished. A healthy system returning the wrong response is `FAIL`; an unavailable required laboratory host before the tested action begins is `BLOCKED`. + + +## 5. Evidence and traceability + +Every logical step must be supported by observed commands and raw output in the native session. Screenshots or other files are attachments, not replacements for command evidence where machine-readable evidence is available. Preserve timestamps, exit codes, stdout, stderr, and tool errors as supplied by the agent CLI. + +Use explicit HTML anchors for all chapters, sections, cases, and steps. Do not rely on renderer-specific heading slugs. `index.json` is the authority for ordering and paths; IDs remain stable after publication. + + +## 6. Compatibility testing + +Derive version combinations from supported deployment behavior rather than testing arbitrary permutations. For a rolling upgrade, cover at least: + +- latest control-plane services with both previous and latest guest images; +- persisted state created by the previous release and consumed by the candidate; +- protocol/schema defaults when one side omits newly introduced fields; +- upgrade order, restart behavior, and rollback where supported; and +- explicit rejection of unsupported combinations with actionable diagnostics. + +For dstack v0.6.0, the expected online topology includes latest VMM, KMS, and gateway components while instances may use a mixture of old and new images. + + +## 7. Release decision + +The final report must provide coverage by requirement and risk, status counts, unresolved failures, blocked or skipped cases, simulation-only results, unconfirmed hardware items, and material deviations from the plan. Release acceptance criteria belong in the plan guide and must state which statuses or open risks block release. + +Before publishing, run `dstack-test validate`, render the self-contained HTML report, and package the selected run. The package is an immutable review artifact and must not include secrets or results from unrelated run IDs. diff --git a/docs/testing/test-case-authoring-spec.md b/docs/testing/test-case-authoring-spec.md new file mode 100644 index 000000000..5b4d8900d --- /dev/null +++ b/docs/testing/test-case-authoring-spec.md @@ -0,0 +1,260 @@ + + + +# dstack Test-Case Authoring Specification + +This document defines the normative layout and content of a dstack test plan and its `case.md` files. See the [test methodology](dstack-test-methodology.md#dstack-test-methodology) and [report output specification](test-report-output-spec.md#dstack-test-report-output-spec). + + +## 1. Plan layout + +```text +/ +├── index.json +├── README.md +├── results// +└── / + ├── README.md # optional + └──
/ + ├── README.md # optional + └── / + ├── case.md + ├── fixtures/ # optional + ├── scripts/ # optional + └── results// +``` + +Only chapter, section, and case are semantic organization levels. Each case has its own directory and a specification named `case.md`. + + +## 2. IDs and anchors + +All referenceable objects use explicit, stable, globally unique ASCII IDs. Use lowercase letters, digits, and hyphens. IDs must not change when titles change. Place `` before each referenceable heading and use relative paths with fragments for cross-file links. + +Recommended forms: + +```text +chapter-gateway +section-gateway-proxy-protocol +tc-gw-pp-001 +tc-gw-pp-001-step-01 +req-gw-pp-001 +risk-gw-spoofing-001 +``` + + +## 3. Top-level guide + +The top-level `README.md` is the first document an executor reads. It must state: + +1. objectives and scope; +2. system topology and system under test; +3. hardware, software, account, and external-service requirements; +4. reproducible common setup commands; +5. shared preconditions and health checks; +6. status rules and release acceptance criteria; +7. evidence, redaction, and attachment rules; +8. order, concurrency, and stop conditions; +9. cleanup and recovery; and +10. validation, packaging, and rendering commands. + +Avoid non-reproducible instructions such as “configure a working KMS.” + + +## 4. `index.json` + +The index is authoritative for discovery and order. A minimal example is: + +```json +{ + "schema_version": "1.0", + "id": "dstack-v0-6-0-release", + "title": "dstack v0.6.0 Release Test Plan", + "guide": {"path": "README.md", "anchor": "release-test-guide"}, + "chapters": [ + { + "id": "chapter-gateway", + "title": "Gateway", + "order": 1, + "path": "01-gateway", + "sections": [ + { + "id": "section-gateway-proxy-protocol", + "title": "Proxy Protocol", + "order": 1, + "path": "01-gateway/01-proxy-protocol", + "cases": [ + { + "id": "tc-gw-pp-001", + "title": "Forward a Proxy v1 client address over TLS termination", + "order": 1, + "priority": "P0", + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001", + "spec": { + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md", + "anchor": "tc-gw-pp-001" + }, + "requirements": ["req-gw-pp-001"], + "risks": ["risk-gw-spoofing-001"], + "tags": ["gateway", "proxy-protocol"] + } + ] + } + ] + } + ] +} +``` + +Array order must agree with `order`. Paths must remain below the plan root and must not contain absolute paths or `..` traversal. + + +## 5. Required `case.md` structure + +Use this order: + +```markdown + +# TC-EXAMPLE-001: Title + +## Metadata + +## Objective + +## Preconditions + +## Test Data + +## Steps + + +### Step 1: Step title + +Action instructions. + +**Expected results:** + +- First observable result. +- Second observable result. + +## Postconditions +``` + + +## 6. Metadata + +At minimum include case ID, priority (`P0`, `P1`, or `P2`), type (for example Functional, Security, Compatibility, Regression, or Performance), minimum environment level, automation suitability, and requirement/risk references. + +Do not repeat common versions in every case. When a case requires an old or special component version, add a **Special Version Requirements** field and record the actual value as a result-level version override. + + +## 7. Objective + +Define one independently decidable behavior: the relevant configuration or state, the action, and the essential externally observable result. Split a case when its title contains multiple independent “and” clauses. An implementation function name or a script's zero exit status is not a product objective. + + +## 8. Preconditions + +Preconditions must be verifiable and distinct from the tested action. Put shared environment conditions in the plan guide and only case-specific conditions in the case. If a prerequisite fails before the tested behavior starts, preserve evidence and report `BLOCKED`; do not report a product `FAIL` for setup failure. + + +## 9. Test data + +Prefer JSON blocks for protocol fields, boundary values, and expected values. Document how random values are generated and preserve the actual values in results. Use RFC documentation address ranges for security-test addresses. Never put reusable credentials, tokens, private keys, or production secrets in plan files. + + +## 10. Steps and expected results + +Prefer three to eight logical steps. One logical step may invoke several mechanical commands, but it validates one phase. Every step must have: + +1. an explicit unique anchor; +2. a reproducible action; +3. precise, observable, comparable expected results; and +4. at least one item of command evidence in the native session. + +Do not write a separate failure criterion. If the actual result does not fully satisfy the expected result, the step and case are `FAIL`. Replace vague words such as “normal,” “correct,” or “without errors” with exact states, fields, addresses, digests, counts, or response codes. + + +## 11. Postconditions + +State which data is removed, which services are stopped or retained, how modified policy/configuration is restored, which state is intentionally retained for later cases, and how cleanup failures are recorded. Cleanup failure does not erase the original test result but must be reported. + + +## 12. Proxy Protocol example + +````markdown + +# TC-GW-PP-001: A PP-enabled application port receives the client address + +## Metadata + +- Priority: P0 +- Type: Functional, Regression, Security +- Environment: INTEGRATION +- Requirements: req-gw-pp-001 +- Risks: risk-gw-spoofing-001 + +## Objective + +Verify that, when inbound Proxy Protocol is enabled and application port 8443 declares `pp=true`, gateway forwards the Proxy v1 client address to the application and completes the following HTTP request. + +## Preconditions + +1. Gateway has `inbound_pp_enabled=true`. +2. The guest is registered and port 8443 has `pp=true`. +3. The capture backend is ready and gateway has loaded the instance port policy. + +## Test Data + +```json +{ + "source": "198.51.100.27:45678", + "destination": "203.0.113.10:8443", + "request_id": "tc-gw-pp-001" +} +``` + + +### Step 1: Check effective policy + +Query the guest and gateway port policy. + +**Expected results:** The instance ID matches; port 8443 exists and has `pp=true`. + + +### Step 2: Reset capture state + +Clear earlier capture records. + +**Expected results:** The backend is ready and contains zero records. + + +### Step 3: Send the request + +Send a Proxy v1 header, then complete TLS and HTTP on the same connection. + +**Expected results:** TLS succeeds, HTTP returns 200, and the request ID matches. + + +### Step 4: Inspect the capture + +Query the backend capture records. + +**Expected results:** Exactly one new record exists; source and destination match the test data and the HTTP request is complete. +```` + + +## 13. Review checklist + +Before submission, confirm that: + +- directory, case ID, and index entry agree; +- explicit anchors are present and unique; +- each case references a requirement or risk; +- the objective has one core behavior; +- preconditions are verifiable; +- every step has precise expected results and raw evidence; +- step count is reasonable; +- simulation and physical-hardware requirements are explicit; +- postconditions restore the environment; and +- no secret or environment-private value is present. diff --git a/docs/testing/test-report-output-spec.md b/docs/testing/test-report-output-spec.md new file mode 100644 index 000000000..d3efcb284 --- /dev/null +++ b/docs/testing/test-report-output-spec.md @@ -0,0 +1,274 @@ + + + +# dstack Test Report Output Specification + +This document defines the normative format for AI sessions, case summaries, run summaries, attachments, cross-references, packages, and self-contained HTML reports. See the [test methodology](dstack-test-methodology.md#dstack-test-methodology) and [case authoring specification](test-case-authoring-spec.md#dstack-test-case-authoring-spec). + + +## 1. Principles + +1. `run-plan` uses one AI orchestration session to drive the `next-case` loop, + and each case it executes runs in an independent Codex or Claude session. +2. The agent's native JSONL is the primary source for commands, tool calls, and raw output. +3. The agent writes one shallow `result.json`; it does not copy command output into that file. +4. The runner generates `runner.json`, timestamps, exit code, checksums, and run aggregates. +5. Common versions and environment information appear once at run level. +6. Screenshots, long logs, and binary captures are separate attachments. +7. Stable anchors make every object linkable in one offline HTML report. + + +## 2. Command interface + +The single public command is `dstack-test`, with consistently named subcommands: + +```text +dstack-test run-case +dstack-test run-plan +dstack-test finalize +dstack-test validate +dstack-test package +dstack-test render +``` + +Options use kebab-case. Execution defaults to Codex; select Claude with +`--agent claude`. Do not introduce separate `--codex` or `--claude` switches. +`run-case` and `run-plan` generate a unique run ID when `--run-id` is omitted. +Commands that operate on an existing run still require its ID. + + +## 3. Case execution + +```bash +dstack-test run-case \ + --plan \ + --case \ + --workdir \ + -- "Additional execution constraints" +``` + +The runner supplies the plan guide, `case.md`, output location, status rules, and result schema in the prompt. The agent must read the guide before the case and must not modify plan specifications. + +For a dependency-driven skip, the orchestrator creates a synthetic one-event +case session containing the reason and causal earlier case IDs. It does not +pretend that the skipped case was executed. The full decision process remains +available in the run-level `orchestrator.jsonl`. + + +## 4. Result layout + +```text +/results// +├── prompt.md +├── session.jsonl +├── agent-stderr.log +├── runner.json +├── result.json +├── artifacts/ +└── SHA256SUMS + +/results// +├── run.json +├── context.json # optional +└── SHA256SUMS +``` + +Separate case directories permit independent sessions. A non-empty result directory must not be overwritten unless the caller explicitly supplies `--overwrite`. + + +## 5. `session.jsonl` + +The runner stores the native agent CLI event stream without rewriting it. Every non-empty line is one complete JSON object. Commands, tool calls, tool results, agent messages, and errors remain in their original format. Renderer adapters normalize Codex and Claude events only for display; the stored file remains unchanged. + +The agent should include the complete step ID when beginning and completing a step: + +```text +tc-gw-pp-001-step-01 +``` + +The renderer links matching session events to the step. If no marker is found, it exposes the complete session as fallback evidence rather than inventing a narrower association. + + +## 6. `runner.json` + +Only `dstack-test` writes this file: + +```json +{ + "schema_version": "1.0", + "run_id": "run-20260723-001", + "case_id": "tc-gw-pp-001", + "agent": {"type": "codex", "model": "gpt-5-codex"}, + "session": { + "format": "codex-jsonl", + "path": "session.jsonl", + "events": 42 + }, + "prompt_path": "prompt.md", + "result_path": "result.json", + "started_at": "2026-07-23T18:00:00.000Z", + "finished_at": "2026-07-23T18:04:32.000Z", + "duration_ms": 272000, + "exit_code": 0, + "result_valid": true, + "result_error": null +} +``` + +Extract the model name from the session when possible, otherwise use explicit `--model`, then `unknown`; never guess. The exit code describes agent infrastructure, not product status. A valid `FAIL` result may accompany a successful agent exit. + + +## 7. Shallow `result.json` + +Before exiting, the agent atomically writes: + +```json +{ + "schema_version": "1.0", + "case_id": "tc-gw-pp-001", + "status": "PASS", + "summary": "Gateway forwarded the Proxy v1 address to the PP-enabled application port.", + "steps": [ + { + "id": "tc-gw-pp-001-step-01", + "status": "PASS", + "observed": "Gateway's cached port 8443 policy had pp=true." + }, + { + "id": "tc-gw-pp-001-step-02", + "status": "PASS", + "observed": "The capture backend was ready and initially contained zero records." + } + ], + "artifacts": [ + { + "name": "Backend capture", + "path": "artifacts/backend-capture.json" + } + ], + "remarks": "" +} +``` + +Constraints: + +- status is `PASS`, `FAIL`, `BLOCKED`, `NOT_RUN`, or `SKIPPED`; +- `PARTIAL` is forbidden; +- `PASS` requires at least one step and all steps must be `PASS`; +- step IDs must come from `case.md`; +- `observed` is a concise observation, not copied raw command output; +- artifact paths must remain inside the result directory; and +- results must not contain tokens, private keys, or other secrets. + + +## 8. Attachments + +Put screenshots, long logs, JSON responses, and binary captures under `artifacts/`; reference them by name and relative path in `result.json`. Generate `SHA256SUMS` during finalization. The renderer displays images inline, shows text and JSON in collapsible blocks, and provides embedded download links for other files. Absolute paths and `..` traversal are forbidden. + + +## 9. Run summary + +After all cases, `dstack-test finalize` scans case outputs and generates `run.json`: + +```json +{ + "schema_version": "1.0", + "id": "run-20260723-001", + "anchor": "run-20260723-001", + "plan_id": "dstack-v0-6-0-release", + "status": "COMPLETED", + "started_at": "2026-07-23T18:00:00.000Z", + "finished_at": "2026-07-23T20:00:00.000Z", + "executors": [{"type": "codex", "model": "gpt-5-codex"}], + "software_under_test": { + "repository": "Dstack-TEE/dstack", + "candidate": "0123456789abcdef", + "previous_release": "v0.5.11" + }, + "environment": {"level": "INTEGRATION", "simulated": true}, + "summary": { + "total": 1, + "completed": 1, + "by_status": { + "PASS": {"count": 1, "case_refs": ["#result-tc-gw-pp-001"]}, + "FAIL": {"count": 0, "case_refs": []}, + "BLOCKED": {"count": 0, "case_refs": []}, + "NOT_RUN": {"count": 0, "case_refs": []}, + "SKIPPED": {"count": 0, "case_refs": []} + } + }, + "case_results": [ + { + "id": "tc-gw-pp-001", + "anchor": "result-tc-gw-pp-001", + "status": "PASS", + "result_path": "../../01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-20260723-001/result.json" + } + ] +} +``` + +Supply common versions and environment data through `--context `. Put exceptional component versions in the relevant case result only. + + +## 10. Validation + +```bash +dstack-test validate --plan --run-id +``` + +Validation must cover at least: + +1. plan paths, IDs, anchors, and index order; +2. one JSON object per non-empty session line; +3. required runner/result files and matching case/run IDs; +4. consistent case and step statuses; +5. safe, existing artifact paths; +6. recomputable run statistics; +7. no missing case in a completed run; and +8. complete and correct `SHA256SUMS` files. + + +## 11. Packaging + +```bash +dstack-test package \ + --plan \ + --run-id \ + --output -.tar.gz +``` + +Supported formats are `.tar.gz`, `.tgz`, `.tar`, and `.zip`. Validate before packaging. Include the complete plan and selected run, but exclude every other historical run stored beside it. + + +## 12. Self-contained HTML + +```bash +dstack-test render \ + --plan \ + --run-id \ + --output report.html +``` + +The output must work offline and inline all CSS, JavaScript, session events, text, JSON, images, and downloadable attachments. It must provide: + +- chapter/section/case navigation; +- the original guide and `case.md` requirements; +- common versions, environment, and executor model; +- status summaries, search, and status filters; +- each case summary and step observation; +- step-to-session-event links; +- collapsible original agent messages, tool calls, command output, and errors; +- every attachment and the complete raw session; and +- stable cross-reference anchors. + + +## 13. Live and historical dashboard + +`run-plan --web` starts a read-only HTTP dashboard for case status and the +native JSONL output of the orchestrator and every case agent. The browser polls +incremental byte ranges so active output appears without rewriting session +files. `dstack-test serve --plan --run-id ` exposes the same view +for a completed or interrupted run. The server has no built-in authentication; +non-loopback binding is permitted only on a trusted network or behind an +authenticated tunnel. diff --git a/tools/dstack-test/README.md b/tools/dstack-test/README.md new file mode 100644 index 000000000..9c8393a5b --- /dev/null +++ b/tools/dstack-test/README.md @@ -0,0 +1,100 @@ + + + +# dstack-test + +`dstack-test` runs each indexed test case in a fresh Codex or Claude session, +preserves the agent's native JSONL event stream, validates shallow case +summaries, materializes run-level statistics, packages results, and renders one +self-contained interactive HTML report. + +## Run one case + +```bash +tools/dstack-test/dstack-test run-case \ + --plan path/to/plan \ + --case tc-gw-pp-001 \ + --workdir "$PWD" \ + -- "Do not restart the physical host" +``` + +The executor defaults to Codex; use `--agent claude` for Claude Code. The run ID +is generated when `--run-id` is omitted. Additional CLI options may be passed +with repeatable `--agent-arg`. The runner writes only below: + +```text +/results// +├── prompt.md +├── session.jsonl +├── agent-stderr.log +├── runner.json +├── result.json # written by the agent +├── artifacts/ +└── SHA256SUMS +``` + +## Run the indexed plan + +```bash +tools/dstack-test/dstack-test run-plan \ + --plan path/to/plan \ + --context run-context.json \ + --web \ + -- "Follow the environment restrictions in README.md" +``` + +`run-plan` starts one orchestration-agent session. The agent drives the loop by +calling `next-case`, followed by either `run-case` or `complete-case`, until the +server reports `COMPLETE`. A file-based control channel lets the outer runner +launch each case agent outside the orchestrator sandbox. When a prior non-PASS +result demonstrably invalidates a later prerequisite, the orchestrator records +that case as `SKIPPED`; it must not skip independent or merely expensive cases. +The complete control conversation is stored in `orchestrator.jsonl`. + +An interrupted run can be continued with the same run ID and `--resume`. + +`--web` starts a read-only live dashboard on `127.0.0.1` and an automatically +selected port. Use `--web-host 0.0.0.0 --web-port 8000` for remote access; +there is no built-in authentication, so expose it only on a trusted network or +through an authenticated tunnel. The dashboard shows case status and polls the +native orchestrator/case JSONL files for near-real-time output. + +After a run has finished, serve the same historical sessions with: + +```bash +tools/dstack-test/dstack-test serve \ + --plan path/to/plan --run-id run-20260723-001 \ + --host 127.0.0.1 --port 8000 +``` + +## Finalize and validate + +```bash +tools/dstack-test/dstack-test finalize \ + --plan path/to/plan --run-id run-20260723-001 \ + --context run-context.json + +tools/dstack-test/dstack-test validate \ + --plan path/to/plan --run-id run-20260723-001 +``` + +## Package and render + +```bash +tools/dstack-test/dstack-test package \ + --plan path/to/plan --run-id run-20260723-001 \ + --output report.tar.gz + +tools/dstack-test/dstack-test render \ + --plan path/to/plan --run-id run-20260723-001 \ + --output report.html +``` + +The renderer uses only the Python standard library. The generated HTML embeds +all CSS, JavaScript, raw session events, text, JSON, images, and downloadable +binary artifacts. It has a chapter/section/case table of contents, status +filters, search, cross-reference anchors, and interactive expansion of raw +session evidence. + +`package` includes the plan and the selected run only; results from other run +IDs are deliberately excluded. diff --git a/tools/dstack-test/dstack-test b/tools/dstack-test/dstack-test new file mode 100755 index 000000000..d9a1a6a89 --- /dev/null +++ b/tools/dstack-test/dstack-test @@ -0,0 +1,1079 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: D100, D101, D102, D103, D107 +"""Run, validate, package, and render dstack test plans.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import secrets +import shutil +import subprocess +import sys +import tarfile +import tempfile +import threading +import time +import zipfile +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import render +import tomllib +import web + +TERMINAL_STATUS = ("PASS", "FAIL", "BLOCKED", "NOT_RUN", "SKIPPED") + + +class DstackTestError(Exception): + """A user-facing dstack-test error.""" + + +def utc_now() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def default_run_id() -> str: + timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + return f"run-{timestamp}-{secrets.token_hex(3)}" + + +def atomic_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", dir=path.parent, delete=False + ) as output: + json.dump(value, output, ensure_ascii=False, indent=2) + output.write("\n") + temporary = Path(output.name) + temporary.replace(path) + + +def find_case(plan: render.Plan, value: str) -> render.CaseEntry: + candidate = Path(value).resolve() + matches = [ + case for case in plan.cases if case.id == value or case.path == candidate + ] + if len(matches) != 1: + raise DstackTestError( + f"expected one indexed case for {value!r}, found {len(matches)}" + ) + return matches[0] + + +def validate_summary(case: render.CaseEntry, path: Path) -> dict[str, Any]: + result = render.load_json(path) + if result.get("case_id") != case.id: + raise DstackTestError(f"result case_id must be {case.id!r}: {path}") + status = result.get("status") + if status not in TERMINAL_STATUS: + raise DstackTestError(f"invalid result status {status!r}: {path}") + if not isinstance(result.get("summary"), str) or not result["summary"].strip(): + raise DstackTestError(f"result summary must be non-empty: {path}") + steps = result.get("steps", []) + if not isinstance(steps, list): + raise DstackTestError(f"result steps must be an array: {path}") + seen: set[str] = set() + for step in steps: + if not isinstance(step, dict): + raise DstackTestError(f"invalid step in {path}") + step_id = step.get("id") + if not isinstance(step_id, str) or not step_id.startswith(case.id + "-step-"): + raise DstackTestError(f"invalid step ID {step_id!r} in {path}") + if step_id in seen: + raise DstackTestError(f"duplicate step ID {step_id!r} in {path}") + seen.add(step_id) + if step.get("status") not in TERMINAL_STATUS: + raise DstackTestError(f"invalid status for {step_id} in {path}") + if not isinstance(step.get("observed"), str) or not step["observed"].strip(): + raise DstackTestError(f"missing observed summary for {step_id} in {path}") + if status == "PASS" and ( + not steps or any(step["status"] != "PASS" for step in steps) + ): + raise DstackTestError( + f"PASS result requires one or more all-PASS steps: {path}" + ) + artifacts = result.get("artifacts", []) + if not isinstance(artifacts, list): + raise DstackTestError(f"artifacts must be an array: {path}") + for artifact in artifacts: + if not isinstance(artifact, dict) or not artifact.get("path"): + raise DstackTestError(f"invalid artifact in {path}") + artifact_path = render.safe_path( + path.parent, artifact["path"], "result artifact" + ) + try: + artifact_path.relative_to(path.parent.resolve()) + except ValueError as error: + raise DstackTestError( + f"artifact escapes result directory: {artifact_path}" + ) from error + if not artifact_path.is_file(): + raise DstackTestError(f"missing result artifact: {artifact_path}") + return result + + +def validate_session(path: Path) -> tuple[int, str | None]: + if not path.is_file() or path.stat().st_size == 0: + raise DstackTestError(f"missing or empty agent session: {path}") + count = 0 + model: str | None = None + with path.open(encoding="utf-8") as source: + for line_no, line in enumerate(source, 1): + if not line.strip(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError as error: + raise DstackTestError( + f"invalid session JSONL at {path}:{line_no}: {error}" + ) from error + if not isinstance(event, dict): + raise DstackTestError( + f"session event is not an object at {path}:{line_no}" + ) + count += 1 + model = model or extract_model(event) + if count == 0: + raise DstackTestError(f"agent session contains no events: {path}") + return count, model + + +def extract_model(value: Any) -> str | None: + if isinstance(value, dict): + for key in ("model", "model_name"): + candidate = value.get(key) + if isinstance(candidate, str) and candidate: + return candidate + for item in value.values(): + if found := extract_model(item): + return found + elif isinstance(value, list): + for item in value: + if found := extract_model(item): + return found + return None + + +def result_schema_text(case_id: str) -> str: + return f'''{{ + "schema_version": "1.0", + "case_id": "{case_id}", + "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", + "summary": "Concise result summary", + "steps": [ + {{"id": "{case_id}-step-01", "status": "PASS|FAIL|BLOCKED|NOT_RUN|SKIPPED", "observed": "Concise observation"}} + ], + "artifacts": [{{"name": "Artifact name", "path": "artifacts/relative-path"}}], + "remarks": "" +}}''' + + +def build_prompt( + plan: render.Plan, case: render.CaseEntry, result_dir: Path, user_prompt: str +) -> str: + return f"""You are executing a test case under the dstack test specification. + +Test plan guide: {plan.guide_path} +Test case: {case.spec_path} +Result directory: {result_dir} +The external runner saves the complete session; do not manually copy command output. + +Execution rules: +1. Read the test plan guide before reading case.md. +2. Do not modify the plan, index.json, case.md, fixtures, or test scripts. +3. Write artifacts only to this result directory or a temporary location explicitly allowed by the guide. +4. Follow case.md step by step. Include the step ID in messages at the start and end of each step so the session can be indexed. +5. Execute actual commands with tools; do not substitute inference or summaries for execution. +6. If an actual result does not fully satisfy the expected result, mark the step and case FAIL. +7. Use BLOCKED when an external prerequisite prevents the tested behavior from starting. Never use PARTIAL. +8. Never present simulation results as physical-hardware results. +9. Do not emit tokens, private keys, key material, or other secrets into the session or attachments. +10. Write screenshots, logs, and other attachments to {result_dir / "artifacts"}. +11. Before finishing, atomically write the summary to {result_dir / "result.json"} and nowhere else. +12. result.json must use this shallow structure and must not embed command output: +{result_schema_text(case.id)} +13. In the final response, state the case status and result.json path. + +Additional caller requirements: +{user_prompt.strip() or "None"} +""" + + +def agent_command( + agent: str, model: str | None, workdir: Path, prompt: str, extra: list[str] +) -> list[str]: + if agent == "codex": + command = ["codex", "exec", "--json", "--color", "never", "-C", str(workdir)] + if model: + command += ["--model", model] + command += extra + command += [prompt] + return command + if agent == "claude": + command = ["claude", "--print", "--verbose", "--output-format", "stream-json"] + if model: + command += ["--model", model] + command += extra + command += [prompt] + return command + raise DstackTestError(f"unsupported agent: {agent}") + + +def resolve_model(agent: str, model: str | None) -> str | None: + if model or agent != "codex": + return model + config_home = Path(os.environ.get("CODEX_HOME", Path.home() / ".codex")) + config_path = config_home / "config.toml" + if not config_path.is_file(): + return None + try: + configured = tomllib.loads(config_path.read_text(encoding="utf-8")).get("model") + except (OSError, tomllib.TOMLDecodeError): + return None + return configured if isinstance(configured, str) and configured else None + + +def run_case( + plan: render.Plan, + case: render.CaseEntry, + run_id: str, + agent: str, + model: str | None, + workdir: Path, + prompt: str, + extra: list[str], + overwrite: bool, +) -> dict[str, Any]: + result_dir = case.path / "results" / run_id + if result_dir.exists() and any(result_dir.iterdir()) and not overwrite: + raise DstackTestError( + f"result directory is not empty; pass --overwrite to replace it: {result_dir}" + ) + if overwrite and result_dir.exists(): + shutil.rmtree(result_dir) + (result_dir / "artifacts").mkdir(parents=True, exist_ok=True) + full_prompt = build_prompt(plan, case, result_dir, prompt) + (result_dir / "prompt.md").write_text(full_prompt, encoding="utf-8") + session_path = result_dir / "session.jsonl" + stderr_path = result_dir / "agent-stderr.log" + started_at = utc_now() + start = time.monotonic_ns() + effective_model = resolve_model(agent, model) + command = agent_command(agent, effective_model, workdir, full_prompt, extra) + binary = shutil.which(command[0]) + if not binary: + raise DstackTestError(f"agent CLI not found: {command[0]}") + with session_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + completed = subprocess.run( + command, cwd=workdir, stdout=stdout, stderr=stderr, check=False + ) + duration_ms = (time.monotonic_ns() - start) // 1_000_000 + finished_at = utc_now() + event_count, detected_model = validate_session(session_path) + result_path = result_dir / "result.json" + result_valid = False + result_error = None + try: + result = validate_summary(case, result_path) + result_valid = True + except (DstackTestError, render.ReportError) as error: + result = None + result_error = str(error) + runner = { + "schema_version": "1.0", + "run_id": run_id, + "case_id": case.id, + "agent": { + "type": agent, + "model": detected_model or effective_model or "unknown", + }, + "session": { + "format": f"{agent}-jsonl", + "path": "session.jsonl", + "events": event_count, + }, + "prompt_path": "prompt.md", + "result_path": "result.json", + "started_at": started_at, + "finished_at": finished_at, + "duration_ms": duration_ms, + "exit_code": completed.returncode, + "result_valid": result_valid, + "result_error": result_error, + } + atomic_json(result_dir / "runner.json", runner) + write_sha256s(result_dir) + if completed.returncode != 0: + raise DstackTestError( + f"{agent} exited with {completed.returncode}; session preserved at {session_path}" + ) + if not result_valid: + raise DstackTestError( + f"agent completed without a valid result.json: {result_error}" + ) + return {"case": case.id, "status": result["status"], "result_dir": str(result_dir)} + + +def skip_case( + case: render.CaseEntry, + run_id: str, + reason: str, + blocked_by: list[str], + model: str | None = None, +) -> dict[str, Any]: + result_dir = case.path / "results" / run_id + if result_dir.exists() and any(result_dir.iterdir()): + raise DstackTestError(f"result directory is not empty: {result_dir}") + result_dir.mkdir(parents=True, exist_ok=True) + now = utc_now() + event = { + "type": "orchestrator.skip", + "case_id": case.id, + "reason": reason, + "blocked_by": blocked_by, + } + (result_dir / "session.jsonl").write_text( + json.dumps(event, ensure_ascii=False) + "\n", encoding="utf-8" + ) + result = { + "schema_version": "1.0", + "case_id": case.id, + "status": "SKIPPED", + "summary": reason, + "steps": [], + "artifacts": [], + "remarks": "Skipped by the run-plan orchestrator after evaluating prior results.", + } + atomic_json(result_dir / "result.json", result) + atomic_json( + result_dir / "runner.json", + { + "schema_version": "1.0", + "run_id": run_id, + "case_id": case.id, + "agent": { + "type": "run-plan-scheduler", + "model": model or "unknown", + }, + "session": { + "format": "orchestrator-decision-jsonl", + "path": "session.jsonl", + "events": 1, + }, + "result_path": "result.json", + "started_at": now, + "finished_at": now, + "duration_ms": 0, + "exit_code": 0, + "result_valid": True, + "result_error": None, + }, + ) + write_sha256s(result_dir) + return {"case": case.id, "status": "SKIPPED", "result_dir": str(result_dir)} + + +def orchestration_prompt(plan: render.Plan, extra_prompt: str) -> str: + return f"""You are the dstack test-plan orchestrator. You drive the entire loop. + +Read {plan.guide_path}, {plan.root / "index.json"}, and all indexed case specifications first. Then repeatedly call `{Path(__file__).resolve()} next-case`. It returns the first unresolved case, its description, and prior results. For each READY case, choose exactly one action: + +- Run it with `{Path(__file__).resolve()} run-case --case `. +- Only when a prior non-PASS result demonstrably invalidates an explicit prerequisite, record it with `{Path(__file__).resolve()} complete-case --case --result SKIPPED --blocked-by --reason `. + +After either action, call next-case again. You may issue the action and next-case as two commands in one shell tool call. Continue until next-case returns COMPLETE. Do not skip for cost, similarity, likely failure, or independently checkable prerequisites. Do not edit plan or result files yourself. Do not call run-plan, finalize, validate, package, or render. + +Additional caller constraints: +{extra_prompt or "None"} +""" + + +class ControlState: + def __init__(self, plan: render.Plan, args: argparse.Namespace, model: str | None): + self.plan, self.args, self.model = plan, args, model + self.lock = threading.Lock() + + def prior(self) -> list[dict[str, Any]]: + values = [] + for case in self.plan.cases: + path = case.path / "results" / self.args.run_id / "result.json" + if path.is_file(): + result = validate_summary(case, path) + values.append( + { + "case_id": case.id, + "status": result["status"], + "summary": result["summary"], + } + ) + return values + + def next_case(self, *, announce: bool = True) -> dict[str, Any]: + prior = self.prior() + done = {item["case_id"] for item in prior} + for number, case in enumerate(self.plan.cases, 1): + if case.id not in done: + if announce: + print( + f"[{number}/{len(self.plan.cases)}] next case: {case.id}", + file=sys.stderr, + flush=True, + ) + return { + "status": "READY", + "position": {"index": number, "total": len(self.plan.cases)}, + "case": { + "id": case.id, + "title": case.title, + "spec_path": str(case.spec_path), + "specification": case.spec_path.read_text(encoding="utf-8"), + }, + "prior_results": prior, + } + return { + "status": "COMPLETE", + "completed": len(done), + "total": len(self.plan.cases), + } + + def current(self, case_id: str) -> tuple[render.CaseEntry, int]: + next_value = self.next_case(announce=False) + if next_value["status"] != "READY" or next_value["case"]["id"] != case_id: + raise DstackTestError(f"{case_id} is not the current unresolved case") + return find_case(self.plan, case_id), next_value["position"]["index"] + + def handle(self, request: dict[str, Any]) -> dict[str, Any]: + with self.lock: + action = request.get("action") + if action == "next-case": + return self.next_case() + case, number = self.current(str(request.get("case"))) + if action == "run-case": + print( + f"[{number}/{len(self.plan.cases)}] executing {case.id}", + file=sys.stderr, + flush=True, + ) + value = run_case( + self.plan, + case, + self.args.run_id, + self.args.agent, + self.model, + self.args.workdir.resolve(), + prompt_text(self.args), + self.args.agent_arg, + False, + ) + print( + f"[{number}/{len(self.plan.cases)}] finished {case.id}: {value['status']}", + file=sys.stderr, + flush=True, + ) + return value + if action == "complete-case": + if request.get("result") != "SKIPPED": + raise DstackTestError( + "complete-case currently accepts only SKIPPED" + ) + blocked_by = request.get("blocked_by", []) + prior = {item["case_id"]: item for item in self.prior()} + if not blocked_by or any( + ref not in prior or prior[ref]["status"] == "PASS" + for ref in blocked_by + ): + raise DstackTestError( + "blocked_by must reference prior non-PASS cases" + ) + value = skip_case( + case, + self.args.run_id, + str(request.get("reason", "")), + blocked_by, + self.model, + ) + print( + f"[{number}/{len(self.plan.cases)}] skipped {case.id}", + file=sys.stderr, + flush=True, + ) + return value + raise DstackTestError(f"unsupported control action: {action}") + + +def control_request(request: dict[str, Any]) -> dict[str, Any]: + control_dir = os.environ.get("DSTACK_TEST_CONTROL_DIR") + if not control_dir: + raise DstackTestError("command requires an active run-plan control server") + root = Path(control_dir) + request_id = secrets.token_hex(12) + request_path = root / f"{request_id}.request.json" + response_path = root / f"{request_id}.response.json" + atomic_json(request_path, request) + deadline = time.monotonic() + 86400 + while not response_path.is_file(): + if time.monotonic() > deadline: + raise DstackTestError("control request timed out") + time.sleep(0.1) + response = render.load_json(response_path) + request_path.unlink(missing_ok=True) + response_path.unlink(missing_ok=True) + if not response.get("ok"): + raise DstackTestError(response.get("error", "control request failed")) + return response["value"] + + +class FileControlServer: + def __init__(self, state: ControlState, root: Path): + self.state, self.root = state, root + self.stop_event = threading.Event() + self.thread = threading.Thread(target=self.serve, daemon=True) + + def start(self) -> None: + self.root.mkdir(parents=True, exist_ok=True) + self.thread.start() + + def serve(self) -> None: + while not self.stop_event.is_set(): + for request_path in sorted(self.root.glob("*.request.json")): + response_path = request_path.with_name( + request_path.name.replace(".request.json", ".response.json") + ) + if response_path.exists(): + continue + try: + value = self.state.handle(render.load_json(request_path)) + response = {"ok": True, "value": value} + except Exception as error: + response = {"ok": False, "error": str(error)} + atomic_json(response_path, response) + self.stop_event.wait(0.05) + + def close(self) -> None: + self.stop_event.set() + self.thread.join() + + +def dashboard_state(plan: render.Plan, run_id: str) -> dict[str, Any]: + run_path = plan.root / "results" / run_id / "run.json" + run = render.load_json(run_path) if run_path.is_file() else {} + counts = {status: 0 for status in (*TERMINAL_STATUS, "RUNNING", "PENDING")} + cases = [] + for case in plan.cases: + result_path = case.path / "results" / run_id / "result.json" + session_path = case.path / "results" / run_id / "session.jsonl" + if result_path.is_file(): + status = str(render.load_json(result_path).get("status", "PENDING")) + else: + status = "RUNNING" if session_path.is_file() else "PENDING" + counts[status] = counts.get(status, 0) + 1 + cases.append({"id": case.id, "title": case.title, "status": status}) + orchestrator = plan.root / "results" / run_id / "orchestrator.jsonl" + return { + "title": plan.index.get("title", plan.index["id"]), + "run_id": run_id, + "run_status": run.get("status", "RUNNING"), + "orchestrator_status": "PASS" + if run_path.is_file() + else ("RUNNING" if orchestrator.is_file() else "PENDING"), + "counts": counts, + "cases": cases, + } + + +def dashboard_log( + plan: render.Plan, run_id: str, agent: str, offset: int +) -> dict[str, Any]: + if agent == "orchestrator": + path = plan.root / "results" / run_id / "orchestrator.jsonl" + elif agent.startswith("case:"): + case = find_case(plan, agent.removeprefix("case:")) + path = case.path / "results" / run_id / "session.jsonl" + else: + raise DstackTestError("unknown agent session") + if not path.is_file(): + return {"text": "", "next_offset": 0, "reset": offset > 0} + size = path.stat().st_size + reset = offset > size + with path.open("rb") as source: + source.seek(0 if reset else offset) + data = source.read(1024 * 1024) + next_offset = source.tell() + return { + "text": data.decode("utf-8", errors="replace"), + "next_offset": next_offset, + "reset": reset, + "eof": next_offset >= size, + } + + +def write_sha256s(root: Path) -> None: + entries: list[str] = [] + for path in sorted( + p for p in root.rglob("*") if p.is_file() and p.name != "SHA256SUMS" + ): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + entries.append(f"{digest} {path.relative_to(root).as_posix()}") + (root / "SHA256SUMS").write_text( + "\n".join(entries) + ("\n" if entries else ""), encoding="utf-8" + ) + + +def verify_sha256s(root: Path) -> None: + sums = root / "SHA256SUMS" + if not sums.is_file(): + raise DstackTestError(f"missing SHA256SUMS: {sums}") + expected_files: set[Path] = set() + for line_no, line in enumerate(sums.read_text(encoding="utf-8").splitlines(), 1): + match = re.fullmatch(r"([0-9a-f]{64}) (.+)", line) + if not match: + raise DstackTestError(f"invalid SHA256SUMS line at {sums}:{line_no}") + path = render.safe_path(root, match.group(2), "SHA256SUMS") + try: + path.relative_to(root.resolve()) + except ValueError as error: + raise DstackTestError( + f"SHA256SUMS path escapes result directory: {path}" + ) from error + if not path.is_file(): + raise DstackTestError(f"SHA256SUMS references missing file: {path}") + if hashlib.sha256(path.read_bytes()).hexdigest() != match.group(1): + raise DstackTestError(f"SHA-256 mismatch: {path}") + expected_files.add(path) + actual_files = { + path.resolve() + for path in root.rglob("*") + if path.is_file() and path.name != "SHA256SUMS" + } + if expected_files != actual_files: + missing = sorted( + str(path.relative_to(root)) for path in actual_files - expected_files + ) + raise DstackTestError(f"files missing from SHA256SUMS in {root}: {missing}") + + +def context_value(path: Path | None) -> dict[str, Any]: + if path is None: + return {} + value = render.load_json(path) + return value + + +def finalize_run( + plan: render.Plan, run_id: str, context: dict[str, Any] | None = None +) -> dict[str, Any]: + case_results: list[dict[str, Any]] = [] + by_status = {status: {"count": 0, "case_refs": []} for status in TERMINAL_STATUS} + started: list[str] = [] + finished: list[str] = [] + executors: list[dict[str, Any]] = [] + incomplete = False + for case in plan.cases: + result_dir = case.path / "results" / run_id + result_path = result_dir / "result.json" + runner_path = result_dir / "runner.json" + if not result_path.is_file() or not runner_path.is_file(): + incomplete = True + continue + result = validate_summary(case, result_path) + runner = render.load_json(runner_path) + validate_session(result_dir / "session.jsonl") + status = result["status"] + by_status[status]["count"] += 1 + by_status[status]["case_refs"].append(f"#result-{case.id}") + case_results.append( + { + "id": case.id, + "anchor": f"result-{case.id}", + "status": status, + "result_path": os.path.relpath( + result_path, plan.root / "results" / run_id + ), + } + ) + if runner.get("started_at"): + started.append(runner["started_at"]) + if runner.get("finished_at"): + finished.append(runner["finished_at"]) + if runner.get("agent") and runner["agent"] not in executors: + executors.append(runner["agent"]) + run_dir = plan.root / "results" / run_id + total = len(plan.cases) + run = { + "schema_version": "1.0", + "id": run_id, + "anchor": run_id, + "plan_id": plan.index["id"], + "status": "INCOMPLETE" + if incomplete or len(case_results) != total + else "COMPLETED", + "started_at": min(started) if started else None, + "finished_at": max(finished) if finished else None, + "executors": executors, + "software_under_test": (context or {}).get("software_under_test", {}), + "environment": (context or {}).get("environment", {}), + "summary": { + "total": total, + "completed": len(case_results), + "by_status": by_status, + }, + "case_results": case_results, + } + atomic_json(run_dir / "run.json", run) + if context: + atomic_json(run_dir / "context.json", context) + write_sha256s(run_dir) + return run + + +def validate_run(plan: render.Plan, run_id: str) -> dict[str, Any]: + run_path = plan.root / "results" / run_id / "run.json" + run = render.load_json(run_path) + if run.get("plan_id") != plan.index["id"] or run.get("id") != run_id: + raise DstackTestError("run/plan ID mismatch") + actual = {status: [] for status in TERMINAL_STATUS} + for item in run.get("case_results", []): + case = find_case(plan, str(item.get("id"))) + result_path = (run_path.parent / str(item.get("result_path", ""))).resolve() + try: + result_path.relative_to(plan.root) + except ValueError as error: + raise DstackTestError( + f"case result escapes plan root: {result_path}" + ) from error + result = validate_summary(case, result_path) + result_dir = result_path.parent + validate_session(result_dir / "session.jsonl") + runner = render.load_json(result_dir / "runner.json") + if not runner.get("result_valid"): + raise DstackTestError(f"runner marks invalid result for {case.id}") + verify_sha256s(result_dir) + actual[result["status"]].append(f"#result-{case.id}") + summary = run.get("summary", {}) + if summary.get("total") != len(plan.cases): + raise DstackTestError("summary total does not match plan") + for status in TERMINAL_STATUS: + item = summary.get("by_status", {}).get(status, {}) + if ( + item.get("count") != len(actual[status]) + or item.get("case_refs") != actual[status] + ): + raise DstackTestError(f"summary mismatch for {status}") + verify_sha256s(run_path.parent) + return { + "status": "valid", + "plan_id": plan.index["id"], + "run_id": run_id, + "cases": len(run.get("case_results", [])), + } + + +def package_plan(plan: render.Plan, run_id: str, output: Path) -> None: + validate_run(plan, run_id) + output.parent.mkdir(parents=True, exist_ok=True) + files = [] + for path in sorted(item for item in plan.root.rglob("*") if item.is_file()): + relative = path.relative_to(plan.root) + parts = relative.parts + # A package is one immutable execution report, not an archive of all + # historical executions that happen to live beside the plan. + if "results" in parts: + index = parts.index("results") + if len(parts) > index + 1 and parts[index + 1] != run_id: + continue + files.append((path, Path("plan") / relative)) + suffix = output.name.lower() + if suffix.endswith(".tar.gz") or suffix.endswith(".tgz"): + with tarfile.open(output, "w:gz") as archive: + for path, archive_path in files: + archive.add(path, arcname=archive_path, recursive=False) + elif suffix.endswith(".tar"): + with tarfile.open(output, "w") as archive: + for path, archive_path in files: + archive.add(path, arcname=archive_path, recursive=False) + elif suffix.endswith(".zip"): + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path, archive_path in files: + archive.write(path, archive_path) + else: + raise DstackTestError("package output must end in .tar.gz, .tgz, .tar, or .zip") + + +def add_common_plan( + parser: argparse.ArgumentParser, + *, + run_id_required: bool = True, + plan_required: bool = True, +) -> None: + parser.add_argument( + "--plan", type=Path, required=plan_required, help="test-plan directory" + ) + parser.add_argument( + "--run-id", + required=run_id_required, + default=None if run_id_required else default_run_id(), + help="shared run identifier" + + ("" if run_id_required else " (default: generated automatically)"), + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="dstack-test", description=__doc__) + sub = parser.add_subparsers(dest="subcommand", required=True) + run_case_parser = sub.add_parser( + "run-case", help="run one indexed case in a fresh AI session" + ) + add_common_plan(run_case_parser, run_id_required=False, plan_required=False) + run_case_parser.add_argument( + "--case", required=True, help="case ID or indexed case directory" + ) + run_case_parser.add_argument( + "--agent", choices=("codex", "claude"), default="codex" + ) + run_case_parser.add_argument("--model") + run_case_parser.add_argument("--workdir", type=Path, default=Path.cwd()) + run_case_parser.add_argument("--prompt-file", type=Path) + run_case_parser.add_argument("--agent-arg", action="append", default=[]) + run_case_parser.add_argument("--overwrite", action="store_true") + run_case_parser.add_argument("prompt", nargs=argparse.REMAINDER) + + run_plan = sub.add_parser( + "run-plan", help="run every indexed case in its own AI session" + ) + add_common_plan(run_plan, run_id_required=False) + run_plan.add_argument("--agent", choices=("codex", "claude"), default="codex") + run_plan.add_argument("--model") + run_plan.add_argument("--workdir", type=Path, default=Path.cwd()) + run_plan.add_argument("--prompt-file", type=Path) + run_plan.add_argument( + "--context", + type=Path, + help="run-level JSON with software_under_test/environment", + ) + run_plan.add_argument("--agent-arg", action="append", default=[]) + run_plan.add_argument("--overwrite", action="store_true") + run_plan.add_argument("--resume", action="store_true") + run_plan.add_argument("--web", action="store_true", help="serve a live dashboard") + run_plan.add_argument("--web-host", default="127.0.0.1") + run_plan.add_argument("--web-port", type=int, default=0) + run_plan.add_argument("prompt", nargs=argparse.REMAINDER) + + sub.add_parser("next-case", help="return the next unresolved controlled case") + complete = sub.add_parser( + "complete-case", help="complete a controlled case without execution" + ) + complete.add_argument("--case", required=True) + complete.add_argument("--result", choices=("SKIPPED",), required=True) + complete.add_argument("--blocked-by", action="append", default=[]) + complete.add_argument("--reason", required=True) + + finalize = sub.add_parser("finalize", help="materialize run.json from case results") + add_common_plan(finalize) + finalize.add_argument("--context", type=Path) + + validate = sub.add_parser("validate", help="validate a finalized test run") + add_common_plan(validate) + + package = sub.add_parser( + "package", help="validate and package a test plan and its results" + ) + add_common_plan(package) + package.add_argument("--output", type=Path, required=True) + + render_parser = sub.add_parser( + "render", help="render a validated run to one offline HTML file" + ) + add_common_plan(render_parser) + render_parser.add_argument("--output", type=Path, required=True) + + serve = sub.add_parser("serve", help="serve a live or historical run dashboard") + add_common_plan(serve) + serve.add_argument("--host", default="127.0.0.1") + serve.add_argument("--port", type=int, default=8000) + return parser + + +def prompt_text(args: argparse.Namespace) -> str: + values = list(args.prompt) + if values and values[0] == "--": + values = values[1:] + text = " ".join(values) + if args.prompt_file: + file_text = args.prompt_file.read_text(encoding="utf-8") + text = f"{file_text}\n\n{text}" if text else file_text + return text + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if os.environ.get("DSTACK_TEST_CONTROL_DIR") and args.subcommand in ( + "next-case", + "run-case", + "complete-case", + ): + if args.subcommand == "next-case": + value = control_request({"action": "next-case"}) + elif args.subcommand == "run-case": + value = control_request({"action": "run-case", "case": args.case}) + else: + value = control_request( + { + "action": "complete-case", + "case": args.case, + "result": args.result, + "blocked_by": args.blocked_by, + "reason": args.reason, + } + ) + print(json.dumps(value, ensure_ascii=False, indent=2)) + return 0 + if not getattr(args, "plan", None): + raise DstackTestError( + "--plan is required outside a controlled run-plan session" + ) + plan = render.load_plan(args.plan) + if args.subcommand == "run-case": + value = run_case( + plan, + find_case(plan, args.case), + args.run_id, + args.agent, + args.model, + args.workdir.resolve(), + prompt_text(args), + args.agent_arg, + args.overwrite, + ) + elif args.subcommand == "run-plan": + run_dir = plan.root / "results" / args.run_id + existing = run_dir.exists() or any( + (case.path / "results" / args.run_id).exists() for case in plan.cases + ) + if existing and not (args.overwrite or args.resume): + raise DstackTestError( + f"run {args.run_id!r} already exists; pass --resume or --overwrite" + ) + if args.overwrite: + shutil.rmtree(run_dir, ignore_errors=True) + for case in plan.cases: + shutil.rmtree(case.path / "results" / args.run_id, ignore_errors=True) + run_dir.mkdir(parents=True, exist_ok=True) + effective_model = resolve_model(args.agent, args.model) + state = ControlState(plan, args, effective_model) + control_dir = run_dir / "control" + server = FileControlServer(state, control_dir) + server.start() + dashboard = None + if args.web: + dashboard = web.Dashboard( + lambda: dashboard_state(plan, args.run_id), + lambda agent, offset: dashboard_log(plan, args.run_id, agent, offset), + args.web_host, + args.web_port, + ) + dashboard.start() + web_host, web_port = dashboard.address + print( + f"dashboard: http://{web_host}:{web_port}/", + file=sys.stderr, + flush=True, + ) + prompt = orchestration_prompt(plan, prompt_text(args)) + (run_dir / "orchestrator-prompt.md").write_text(prompt, encoding="utf-8") + environment = os.environ.copy() + environment["DSTACK_TEST_CONTROL_DIR"] = str(control_dir) + command = agent_command( + args.agent, effective_model, args.workdir.resolve(), prompt, args.agent_arg + ) + try: + with ( + (run_dir / "orchestrator.jsonl").open("wb") as stdout, + (run_dir / "orchestrator-stderr.log").open("wb") as stderr, + ): + completed = subprocess.run( + command, + cwd=args.workdir, + env=environment, + stdout=stdout, + stderr=stderr, + check=False, + ) + finally: + server.close() + shutil.rmtree(control_dir, ignore_errors=True) + if dashboard: + dashboard.close() + events, detected_model = validate_session(run_dir / "orchestrator.jsonl") + atomic_json( + run_dir / "orchestrator-runner.json", + { + "schema_version": "1.0", + "agent": { + "type": args.agent, + "model": detected_model or effective_model or "unknown", + }, + "events": events, + "exit_code": completed.returncode, + }, + ) + run = finalize_run(plan, args.run_id, context_value(args.context)) + value = {"orchestrator_exit_code": completed.returncode, "run": run} + elif args.subcommand == "finalize": + value = finalize_run(plan, args.run_id, context_value(args.context)) + elif args.subcommand == "validate": + value = validate_run(plan, args.run_id) + elif args.subcommand == "package": + package_plan(plan, args.run_id, args.output) + value = { + "status": "packaged", + "output": str(args.output), + "sha256": hashlib.sha256(args.output.read_bytes()).hexdigest(), + } + elif args.subcommand == "render": + validate_run(plan, args.run_id) + run, results = render.load_session_results(plan, args.run_id) + rendered = render.render_report(plan, run, results) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + value = { + "status": "rendered", + "output": str(args.output), + "bytes": args.output.stat().st_size, + } + elif args.subcommand == "serve": + dashboard = web.Dashboard( + lambda: dashboard_state(plan, args.run_id), + lambda agent, offset: dashboard_log(plan, args.run_id, agent, offset), + args.host, + args.port, + ) + dashboard.start() + host, port = dashboard.address + print(f"dashboard: http://{host}:{port}/", file=sys.stderr, flush=True) + try: + dashboard.thread.join() + except KeyboardInterrupt: + dashboard.close() + return 0 + else: + raise AssertionError(args.subcommand) + print(json.dumps(value, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (DstackTestError, render.ReportError) as error: + print( + json.dumps({"status": "error", "message": str(error)}, ensure_ascii=False), + file=sys.stderr, + ) + raise SystemExit(2) diff --git a/tools/dstack-test/render.py b/tools/dstack-test/render.py new file mode 100755 index 000000000..739ef4bba --- /dev/null +++ b/tools/dstack-test/render.py @@ -0,0 +1,932 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: D100, D101, D102, D103 +"""Validate and render a dstack test-plan run as one self-contained HTML file.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import html +import json +import mimetypes +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +STATUS = ("PASS", "FAIL", "BLOCKED", "NOT_RUN", "SKIPPED") +ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +ANCHOR_LINE_RE = re.compile(r'^\s*\s*$') +HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$") +LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") +CODE_RE = re.compile(r"`([^`]+)`") +BOLD_RE = re.compile(r"\*\*([^*]+)\*\*") + + +class ReportError(Exception): + """Raised when a plan or result violates the report contract.""" + + +@dataclass +class CaseEntry: + chapter_id: str + chapter_title: str + section_id: str + section_title: str + id: str + title: str + path: Path + spec_path: Path + spec_anchor: str + priority: str = "" + requirements: list[str] = field(default_factory=list) + risks: list[str] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + + +@dataclass +class Plan: + root: Path + index: dict[str, Any] + guide_path: Path + cases: list[CaseEntry] + chapter_docs: dict[str, Path] + section_docs: dict[str, Path] + anchors: set[str] + + +def load_json(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as error: + raise ReportError(f"missing JSON file: {path}") from error + except json.JSONDecodeError as error: + raise ReportError(f"invalid JSON in {path}: {error}") from error + if not isinstance(value, dict): + raise ReportError(f"expected JSON object in {path}") + return value + + +def require(obj: dict[str, Any], key: str, context: str) -> Any: + if key not in obj: + raise ReportError(f"missing {key!r} in {context}") + return obj[key] + + +def check_id(value: Any, context: str) -> str: + if not isinstance(value, str) or not ID_RE.fullmatch(value): + raise ReportError(f"invalid ID {value!r} in {context}") + return value + + +def safe_path(root: Path, relative: str, context: str) -> Path: + if not isinstance(relative, str) or not relative or Path(relative).is_absolute(): + raise ReportError(f"invalid relative path {relative!r} in {context}") + candidate = (root / relative).resolve() + try: + candidate.relative_to(root.resolve()) + except ValueError as error: + raise ReportError(f"path escapes plan root in {context}: {relative}") from error + return candidate + + +def markdown_anchors(path: Path) -> set[str]: + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as error: + raise ReportError(f"missing Markdown document: {path}") from error + return { + match.group(1) + for line in text.splitlines() + if (match := ANCHOR_LINE_RE.match(line)) + } + + +def sorted_by_order(items: Any, context: str) -> list[dict[str, Any]]: + if not isinstance(items, list): + raise ReportError(f"expected array in {context}") + result: list[dict[str, Any]] = [] + orders: list[int] = [] + for item in items: + if not isinstance(item, dict): + raise ReportError(f"expected object in {context}") + order = require(item, "order", context) + if not isinstance(order, int) or order < 1: + raise ReportError(f"invalid order in {context}: {order!r}") + result.append(item) + orders.append(order) + if orders != sorted(orders) or len(orders) != len(set(orders)): + raise ReportError(f"items in {context} must have unique ascending order") + return result + + +def load_plan(root: Path) -> Plan: + root = root.resolve() + index = load_json(root / "index.json") + plan_id = check_id(require(index, "id", "index.json"), "index.json") + guide = require(index, "guide", "index.json") + if not isinstance(guide, dict): + raise ReportError("guide must be an object") + guide_path = safe_path(root, require(guide, "path", "guide"), "guide") + guide_anchor = check_id(require(guide, "anchor", "guide"), "guide") + anchors: set[str] = {plan_id} + guide_anchors = markdown_anchors(guide_path) + if guide_anchor not in guide_anchors: + raise ReportError(f"guide anchor #{guide_anchor} not found in {guide_path}") + anchors.update(guide_anchors) + + cases: list[CaseEntry] = [] + chapter_docs: dict[str, Path] = {} + section_docs: dict[str, Path] = {} + ids = {plan_id} + for chapter in sorted_by_order( + require(index, "chapters", "index.json"), "chapters" + ): + chapter_id = check_id(require(chapter, "id", "chapter"), "chapter") + if chapter_id in ids: + raise ReportError(f"duplicate ID: {chapter_id}") + ids.add(chapter_id) + chapter_title = str(require(chapter, "title", chapter_id)) + chapter_root = safe_path(root, require(chapter, "path", chapter_id), chapter_id) + document = chapter.get("document") + if isinstance(document, dict): + doc = safe_path(root, require(document, "path", chapter_id), chapter_id) + anchor = check_id(require(document, "anchor", chapter_id), chapter_id) + doc_anchors = markdown_anchors(doc) + if anchor not in doc_anchors: + raise ReportError(f"chapter anchor #{anchor} not found in {doc}") + anchors.update(doc_anchors) + chapter_docs[chapter_id] = doc + elif (chapter_root / "README.md").is_file(): + chapter_docs[chapter_id] = chapter_root / "README.md" + anchors.update(markdown_anchors(chapter_root / "README.md")) + + for section in sorted_by_order( + require(chapter, "sections", chapter_id), f"{chapter_id}.sections" + ): + section_id = check_id(require(section, "id", "section"), "section") + if section_id in ids: + raise ReportError(f"duplicate ID: {section_id}") + ids.add(section_id) + section_title = str(require(section, "title", section_id)) + section_root = safe_path( + root, require(section, "path", section_id), section_id + ) + document = section.get("document") + if isinstance(document, dict): + doc = safe_path(root, require(document, "path", section_id), section_id) + anchor = check_id(require(document, "anchor", section_id), section_id) + doc_anchors = markdown_anchors(doc) + if anchor not in doc_anchors: + raise ReportError(f"section anchor #{anchor} not found in {doc}") + anchors.update(doc_anchors) + section_docs[section_id] = doc + elif (section_root / "README.md").is_file(): + section_docs[section_id] = section_root / "README.md" + anchors.update(markdown_anchors(section_root / "README.md")) + + for case in sorted_by_order( + require(section, "cases", section_id), f"{section_id}.cases" + ): + case_id = check_id(require(case, "id", "case"), "case") + if case_id in ids: + raise ReportError(f"duplicate ID: {case_id}") + ids.add(case_id) + case_path = safe_path(root, require(case, "path", case_id), case_id) + spec = require(case, "spec", case_id) + if not isinstance(spec, dict): + raise ReportError(f"spec must be object in {case_id}") + spec_path = safe_path(root, require(spec, "path", case_id), case_id) + spec_anchor = check_id(require(spec, "anchor", case_id), case_id) + case_anchors = markdown_anchors(spec_path) + if spec_anchor not in case_anchors: + raise ReportError( + f"spec anchor #{spec_anchor} not found in {spec_path}" + ) + duplicate_anchors = anchors.intersection(case_anchors) + if duplicate_anchors: + raise ReportError( + f"duplicate Markdown anchor(s): {sorted(duplicate_anchors)}" + ) + anchors.update(case_anchors) + cases.append( + CaseEntry( + chapter_id=chapter_id, + chapter_title=chapter_title, + section_id=section_id, + section_title=section_title, + id=case_id, + title=str(require(case, "title", case_id)), + path=case_path, + spec_path=spec_path, + spec_anchor=spec_anchor, + priority=str(case.get("priority", "")), + requirements=list(case.get("requirements", [])), + risks=list(case.get("risks", [])), + tags=list(case.get("tags", [])), + ) + ) + return Plan(root, index, guide_path, cases, chapter_docs, section_docs, anchors) + + +def inline_markup(text: str) -> str: + escaped = html.escape(text, quote=False) + escaped = CODE_RE.sub(lambda m: f"{m.group(1)}", escaped) + escaped = BOLD_RE.sub(lambda m: f"{m.group(1)}", escaped) + + def link(match: re.Match[str]) -> str: + label, target = match.group(1), html.unescape(match.group(2)) + if "#" in target: + target = "#" + target.rsplit("#", 1)[1] + return f'{label}' + + return LINK_RE.sub(link, escaped) + + +def markdown_to_html(text: str) -> str: + """Render the deliberately small Markdown subset used by test plans.""" + lines = text.splitlines() + out: list[str] = [] + paragraph: list[str] = [] + list_type: str | None = None + in_code = False + code_lang = "" + code_lines: list[str] = [] + table_lines: list[str] = [] + + def flush_paragraph() -> None: + nonlocal paragraph + if paragraph: + out.append( + f"

{inline_markup(' '.join(x.strip() for x in paragraph))}

" + ) + paragraph = [] + + def close_list() -> None: + nonlocal list_type + if list_type: + out.append(f"") + list_type = None + + def flush_table() -> None: + nonlocal table_lines + if not table_lines: + return + rows = [ + [cell.strip() for cell in line.strip().strip("|").split("|")] + for line in table_lines + ] + if len(rows) >= 2 and all( + re.fullmatch(r":?-{3,}:?", c.replace(" ", "")) for c in rows[1] + ): + out.append( + '
' + + "".join(f"" for c in rows[0]) + + "" + ) + for row in rows[2:]: + out.append( + "" + + "".join(f"" for c in row) + + "" + ) + out.append("
{inline_markup(c)}
{inline_markup(c)}
") + else: + out.extend(f"

{inline_markup(line)}

" for line in table_lines) + table_lines = [] + + for line in lines + [""]: + if in_code: + if line.startswith("```"): + out.append( + f'
{html.escape(chr(10).join(code_lines))}
' + ) + in_code = False + code_lines = [] + else: + code_lines.append(line) + continue + if line.startswith("```"): + flush_paragraph() + close_list() + flush_table() + in_code = True + code_lang = line[3:].strip() + continue + if match := ANCHOR_LINE_RE.match(line): + flush_paragraph() + close_list() + flush_table() + out.append(f'') + continue + if match := HEADING_RE.match(line): + flush_paragraph() + close_list() + flush_table() + level = len(match.group(1)) + out.append(f"{inline_markup(match.group(2))}") + continue + if line.startswith("|") and line.rstrip().endswith("|"): + flush_paragraph() + close_list() + table_lines.append(line) + continue + flush_table() + if match := re.match(r"^\s*[-*]\s+(.+)$", line): + flush_paragraph() + if list_type != "ul": + close_list() + out.append("
    ") + list_type = "ul" + out.append(f"
  • {inline_markup(match.group(1))}
  • ") + continue + if match := re.match(r"^\s*\d+[.)]\s+(.+)$", line): + flush_paragraph() + if list_type != "ol": + close_list() + out.append("
      ") + list_type = "ol" + out.append(f"
    1. {inline_markup(match.group(1))}
    2. ") + continue + if not line.strip(): + flush_paragraph() + close_list() + else: + close_list() + paragraph.append(line) + return "\n".join(out) + + +def status_badge(status: str) -> str: + cls = status.lower().replace("_", "-") + return f'{html.escape(status)}' + + +def json_block(value: Any) -> str: + return f'
      {html.escape(json.dumps(value, ensure_ascii=False, indent=2))}
      ' + + +def ref_link(ref: str, label: str | None = None) -> str: + fragment = ref.rsplit("#", 1)[-1] if "#" in ref else "" + href = f"#{html.escape(fragment, quote=True)}" if fragment else "#" + return f'{html.escape(label or ref)}' + + +def resolve_ref_file(base: Path, ref: str, plan_root: Path) -> tuple[Path, str]: + path_part, _, anchor = ref.partition("#") + path = base if not path_part else (base.parent / path_part).resolve() + try: + path.relative_to(plan_root) + except ValueError as error: + raise ReportError(f"reference escapes plan root: {ref}") from error + if not path.is_file(): + raise ReportError(f"referenced file does not exist: {ref} from {base}") + return path, anchor + + +def load_evidence( + result_path: Path, ref: str, plan_root: Path +) -> tuple[dict[str, Any], Path]: + path, anchor = resolve_ref_file(result_path, ref, plan_root) + value = load_json(path) + if anchor and anchor not in (value.get("anchor"), value.get("id")): + raise ReportError(f"evidence anchor #{anchor} not found in {path}") + return value, path + + +def verify_attachment(result_path: Path, item: dict[str, Any], plan_root: Path) -> Path: + path = safe_path( + result_path.parent, require(item, "path", "attachment"), "attachment" + ) + try: + path.relative_to(plan_root) + except ValueError as error: + raise ReportError(f"attachment escapes plan root: {path}") from error + if not path.is_file(): + raise ReportError(f"missing attachment: {path}") + data = path.read_bytes() + expected_size = item.get("size_bytes") + if expected_size is not None and expected_size != len(data): + raise ReportError(f"attachment size mismatch: {path}") + expected_hash = item.get("sha256") + if expected_hash and hashlib.sha256(data).hexdigest() != expected_hash: + raise ReportError(f"attachment SHA-256 mismatch: {path}") + return path + + +def render_command(command: dict[str, Any], anchor_override: str | None = None) -> str: + anchor = anchor_override or str( + command.get("anchor") or command.get("id") or "command" + ) + title = str(command.get("command", "command")) + env = command.get("environment", {}) + meta = { + "cwd": command.get("cwd"), + "environment": env, + "started_at": command.get("started_at"), + "finished_at": command.get("finished_at"), + "duration_ms": command.get("duration_ms"), + "exit_code": command.get("exit_code"), + } + parts = [ + f'', + '
      ', + f"{html.escape(title)}", + f' exit {html.escape(str(command.get("exit_code", "?")))}', + ] + parts.append('
      Execution metadata
      ') + parts.append(json_block(meta)) + for stream in ("stdout", "stderr"): + value = command.get(stream) + if value is not None: + parts.append( + f'
      {stream}
      {html.escape(str(value))}
      ' + ) + attachment = command.get(f"{stream}_attachment") + if attachment: + parts.append( + f"

      {stream} attachment: {ref_link(str(attachment.get('ref', '')))}

      " + ) + parts.append("
      ") + return "".join(parts) + + +def render_attachment(item: dict[str, Any], path: Path) -> str: + data = path.read_bytes() + media = str( + item.get("media_type") + or mimetypes.guess_type(path.name)[0] + or "application/octet-stream" + ) + anchor = str(item.get("anchor") or item.get("id") or f"attachment-{path.name}") + title = str(item.get("title") or path.name) + digest = hashlib.sha256(data).hexdigest() + header = f'
      {html.escape(title)}

      {html.escape(media)} · {len(data)} bytes · SHA-256 {digest}

      ' + if media.startswith("image/"): + uri = f"data:{media};base64,{base64.b64encode(data).decode()}" + return ( + header + + f'{html.escape(title)}' + ) + if media.startswith("text/") or media in ("application/json", "application/xml"): + text = data.decode("utf-8", errors="replace") + lang = "json" if media == "application/json" else "text" + return ( + header + + f'
      View {html.escape(path.name)}
      {html.escape(text)}
      ' + ) + uri = f"data:{media};base64,{base64.b64encode(data).decode()}" + return ( + header + + f'Download inline attachment' + ) + + +def validate_case_result(case: CaseEntry, result: dict[str, Any], path: Path) -> None: + if result.get("id") != case.id: + raise ReportError(f"case ID mismatch in {path}: expected {case.id}") + status = result.get("status") + if status not in STATUS: + raise ReportError(f"invalid case status {status!r} in {path}") + steps = result.get("steps", []) + if not isinstance(steps, list): + raise ReportError(f"steps must be array in {path}") + for step in steps: + if not isinstance(step, dict) or step.get("status") not in STATUS: + raise ReportError(f"invalid step in {path}") + refs = step.get("evidence_refs", []) + commands = step.get("commands", []) + if status not in ("NOT_RUN", "SKIPPED") and not refs and not commands: + raise ReportError( + f"step {step.get('id')} has no command evidence in {path}" + ) + if status == "PASS" and any(step.get("status") != "PASS" for step in steps): + raise ReportError(f"PASS case contains non-PASS step in {path}") + if status == "PASS" and not steps: + raise ReportError(f"PASS case has no steps in {path}") + + +def session_events(path: Path) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as source: + for line_no, line in enumerate(source, 1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as error: + raise ReportError( + f"invalid session JSONL at {path}:{line_no}: {error}" + ) from error + if not isinstance(value, dict): + raise ReportError(f"session event is not an object at {path}:{line_no}") + events.append(value) + if not events: + raise ReportError(f"empty session JSONL: {path}") + return events + + +def event_title(event: dict[str, Any], number: int) -> str: + kind = str( + event.get("type") or event.get("event") or event.get("subtype") or "event" + ) + item = event.get("item") + if isinstance(item, dict) and item.get("type"): + kind += f" · {item['type']}" + message = event.get("message") + if isinstance(message, dict): + content = message.get("content") + if isinstance(content, str) and content.strip(): + kind += f" · {content.strip()[:80]}" + return f"{number:04d} · {kind}" + + +def render_session(case_id: str, events: list[dict[str, Any]]) -> str: + parts = [ + '
      Complete agent session (' + + str(len(events)) + + ' events)
      ' + ] + for number, event in enumerate(events, 1): + anchor = f"session-{case_id}-event-{number}" + parts.append( + f'
      {html.escape(event_title(event, number))}{json_block(event)}
      ' + ) + parts.append("
      ") + return "".join(parts) + + +def step_session_ref(step_id: str, events: list[dict[str, Any]], case_id: str) -> str: + for number, event in enumerate(events, 1): + if step_id in json.dumps(event, ensure_ascii=False): + return f'Jump to session evidence · event {number}' + return 'Evidence mapping: whole session fallback' + + +def render_case_result( + case: CaseEntry, result: dict[str, Any], result_path: Path, plan: Plan +) -> str: + status = str(result["status"]) + result_dir = result_path.parent + runner_path = result_dir / "runner.json" + session_path = result_dir / "session.jsonl" + if not runner_path.is_file(): + raise ReportError(f"missing runner.json for {case.id}") + runner = load_json(runner_path) + events = session_events(session_path) + parts = [ + f'
      ' + ] + parts.append( + f'

      Execution result

      {status_badge(status)}
      ' + ) + parts.append( + f'
      {html.escape(str(result.get("summary", "")))}
      ' + ) + parts.append( + '
      Runner metadata' + + json_block(runner) + + "
      " + ) + if result.get("version_overrides"): + parts.append( + '
      Version overrides' + + json_block(result["version_overrides"]) + + "
      " + ) + for step in result.get("steps", []): + step_id = str(step.get("id", "step")) + step_status = str(step.get("status", "")) + parts.append( + f'

      {html.escape(step_id)}

      {status_badge(step_status)}
      ' + ) + parts.append( + f'
      Observed

      {html.escape(str(step.get("observed", "")))}

      ' + ) + parts.append(step_session_ref(step_id, events, case.id)) + parts.append("
      ") + artifacts = result.get("artifacts", []) + if artifacts: + parts.append( + '
      Artifacts' + ) + for number, artifact in enumerate(artifacts, 1): + if not isinstance(artifact, dict) or not artifact.get("path"): + raise ReportError(f"invalid artifact in {result_path}") + path = safe_path(result_dir, str(artifact["path"]), "result artifact") + try: + path.relative_to(result_dir.resolve()) + except ValueError as error: + raise ReportError( + f"artifact escapes result directory: {path}" + ) from error + if not path.is_file(): + raise ReportError(f"missing artifact: {path}") + item = { + "id": f"attachment-{case.id}-{number}", + "anchor": f"attachment-{case.id}-{number}", + "title": artifact.get("name") or path.name, + "media_type": artifact.get("media_type") + or mimetypes.guess_type(path.name)[0], + } + parts.append(render_attachment(item, path)) + parts.append("
      ") + parts.append(render_session(case.id, events)) + if result.get("remarks"): + parts.append( + f'
      Remarks: {html.escape(str(result["remarks"]))}
      ' + ) + parts.append("
      ") + return "".join(parts) + + +def load_session_results( + plan: Plan, run_id: str +) -> tuple[dict[str, Any], dict[str, tuple[dict[str, Any], Path]]]: + run_path = plan.root / "results" / run_id / "run.json" + run = load_json(run_path) + if run.get("id") != run_id or run.get("plan_id") != plan.index.get("id"): + raise ReportError(f"run/plan ID mismatch in {run_path}") + listed = run.get("case_results", []) + if not isinstance(listed, list): + raise ReportError("case_results must be an array") + path_by_id: dict[str, Path] = {} + for item in listed: + if not isinstance(item, dict): + raise ReportError("case_results entries must be objects") + case_id = check_id(require(item, "id", "case_results"), "case_results") + candidate = ( + run_path.parent / str(require(item, "result_path", case_id)) + ).resolve() + try: + candidate.relative_to(plan.root) + except ValueError as error: + raise ReportError(f"case result escapes plan root: {candidate}") from error + path_by_id[case_id] = candidate + results: dict[str, tuple[dict[str, Any], Path]] = {} + for case in plan.cases: + path = path_by_id.get(case.id) + if path is None or not path.is_file(): + raise ReportError(f"missing finalized result for {case.id}") + result = load_json(path) + if result.get("case_id") != case.id or result.get("status") not in STATUS: + raise ReportError(f"invalid shallow result for {case.id}: {path}") + session_events(path.parent / "session.jsonl") + if not (path.parent / "runner.json").is_file(): + raise ReportError(f"missing runner.json for {case.id}") + results[case.id] = (result, path) + return run, results + + +def load_results( + plan: Plan, run_id: str +) -> tuple[dict[str, Any], dict[str, tuple[dict[str, Any], Path]]]: + run_path = plan.root / "results" / run_id / "run.json" + run = load_json(run_path) + if run.get("id") != run_id: + raise ReportError(f"run ID mismatch in {run_path}") + if run.get("plan_id") != plan.index.get("id"): + raise ReportError(f"plan ID mismatch in {run_path}") + listed = run.get("case_results", []) + path_by_id: dict[str, Path] = {} + if not isinstance(listed, list): + raise ReportError("case_results must be an array") + for item in listed: + if not isinstance(item, dict): + raise ReportError("case_results entries must be objects") + case_id = check_id(require(item, "id", "case_results"), "case_results") + candidate = ( + run_path.parent / str(require(item, "result_path", case_id)) + ).resolve() + try: + candidate.relative_to(plan.root) + except ValueError as error: + raise ReportError(f"case result escapes plan root: {candidate}") from error + path_by_id[case_id] = candidate + results: dict[str, tuple[dict[str, Any], Path]] = {} + for case in plan.cases: + path = path_by_id.get(case.id, case.path / "results" / run_id / "result.json") + if not path.is_file(): + raise ReportError(f"missing result for {case.id}: {path}") + result = load_json(path) + validate_case_result(case, result, path) + results[case.id] = (result, path) + actual: dict[str, list[str]] = {status: [] for status in STATUS} + for case in plan.cases: + actual[str(results[case.id][0]["status"])].append(f"#result-{case.id}") + summary = require(run, "summary", str(run_path)) + if not isinstance(summary, dict) or summary.get("total") != len(plan.cases): + raise ReportError("run summary total does not match plan") + by_status = require(summary, "by_status", "summary") + for status in STATUS: + item = by_status.get(status, {}) + if ( + item.get("count") != len(actual[status]) + or item.get("case_refs") != actual[status] + ): + raise ReportError(f"summary mismatch for {status}") + return run, results + + +def nav_html(plan: Plan, results: dict[str, tuple[dict[str, Any], Path]]) -> str: + guide_anchor = html.escape(str(plan.index["guide"]["anchor"]), quote=True) + parts = [ + f'") + return "".join(parts) + + +CSS = r""" +:root{--bg:#f5f7fb;--panel:#fff;--ink:#172033;--muted:#667085;--line:#e3e8ef;--brand:#5b5bd6;--brand2:#7c3aed;--pass:#067647;--pass-bg:#ecfdf3;--fail:#b42318;--fail-bg:#fef3f2;--blocked:#b54708;--blocked-bg:#fffaeb;--not:#475467;--not-bg:#f2f4f7;--shadow:0 10px 30px rgba(16,24,40,.08)} +*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.65 Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif}.layout{display:grid;grid-template-columns:300px minmax(0,1fr);min-height:100vh}.sidebar{position:sticky;top:0;height:100vh;overflow:auto;background:#111827;color:#e5e7eb;padding:22px 16px}.brand{font-size:19px;font-weight:750;margin-bottom:4px}.run-label{font-size:12px;color:#9ca3af;margin-bottom:18px}.search{width:100%;border:1px solid #374151;background:#1f2937;color:#fff;border-radius:9px;padding:9px 11px;margin-bottom:12px}.filter-row{display:flex;gap:5px;flex-wrap:wrap;margin-bottom:14px}.filter{border:1px solid #374151;background:#1f2937;color:#d1d5db;border-radius:999px;padding:4px 8px;font-size:11px;cursor:pointer}.filter.active{background:var(--brand);border-color:var(--brand);color:#fff}.toc a{color:#d1d5db;text-decoration:none;display:flex;align-items:center;justify-content:space-between;padding:5px 7px;border-radius:6px}.toc a:hover{background:#1f2937;color:#fff}.toc-title{text-transform:uppercase;color:#9ca3af;font-size:11px;letter-spacing:.1em;margin:10px 7px}.toc-chapter{margin-top:8px}.toc-chapter>a{font-weight:700}.toc-section{margin-left:9px}.toc-section>a{font-size:13px;font-weight:600}.toc-case{margin-left:12px;font-size:12px;gap:6px}.toc .status{font-size:9px;padding:1px 5px}.main{min-width:0;padding:36px 46px 80px;max-width:1440px;width:100%;margin:auto}.hero{background:linear-gradient(125deg,#312e81,#6d28d9 60%,#2563eb);color:#fff;padding:35px;border-radius:18px;box-shadow:var(--shadow);margin-bottom:24px}.hero h1{margin:0 0 6px;font-size:30px}.hero p{color:#ddd6fe;margin:0}.panel,.chapter,.section,.case-card{background:var(--panel);border:1px solid var(--line);border-radius:14px;box-shadow:0 3px 14px rgba(16,24,40,.04);padding:24px;margin:18px 0}.chapter{border-left:5px solid var(--brand)}.section{margin-left:12px;border-left:4px solid #a78bfa}.case-card{margin-left:24px;padding:0;overflow:hidden}.case-spec{padding:26px}.case-titlebar{display:flex;gap:12px;align-items:center;justify-content:space-between}.case-result{padding:24px;border-top:1px solid var(--line);background:#fcfcfd}.status-border-pass{border-left:5px solid var(--pass)}.status-border-fail{border-left:5px solid var(--fail)}.status-border-blocked{border-left:5px solid var(--blocked)}.status-border-not-run,.status-border-skipped{border-left:5px solid var(--not)}h1,h2,h3,h4,h5{line-height:1.25;scroll-margin-top:20px}h2{margin-top:30px}h3{margin-top:24px}a{color:#4f46e5}.anchor{scroll-margin-top:18px}.status{display:inline-flex;align-items:center;border-radius:999px;padding:3px 9px;font-size:11px;font-weight:750;letter-spacing:.03em}.pass{color:var(--pass);background:var(--pass-bg)}.fail{color:var(--fail);background:var(--fail-bg)}.blocked{color:var(--blocked);background:var(--blocked-bg)}.not-run,.skipped{color:var(--not);background:var(--not-bg)}.summary-grid{display:grid;grid-template-columns:repeat(5,minmax(90px,1fr));gap:12px;margin:18px 0}.summary-card{padding:15px;border:1px solid var(--line);border-radius:11px;background:#fff}.summary-card strong{display:block;font-size:25px}.summary-card span{color:var(--muted);font-size:12px}.table-wrap{overflow:auto}table{border-collapse:collapse;width:100%;margin:12px 0}th,td{border:1px solid var(--line);padding:9px 11px;text-align:left;vertical-align:top}th{background:#f8fafc}code{font-family:"SFMono-Regular",Consolas,monospace;font-size:.9em;background:#f2f4f7;border-radius:5px;padding:1px 4px}pre{overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere;background:#101828;color:#e5e7eb;border-radius:10px;padding:14px;font:12px/1.55 "SFMono-Regular",Consolas,monospace}pre code{background:none;padding:0}.json{background:#172033}.stderr{border-left:4px solid #f04438}.stdout{border-left:4px solid #12b76a}details{border:1px solid var(--line);border-radius:10px;margin:10px 0;background:#fff}summary{cursor:pointer;padding:10px 13px;font-weight:650;list-style:none}summary::-webkit-details-marker{display:none}.chevron{display:inline-block;margin-right:7px;transition:transform .18s}details[open]>summary .chevron{transform:rotate(90deg)}details>pre,details>.json,details>.command-body,details>.assertion-grid,details>.attachment-meta{margin:0 12px 12px}.case-result-head,.step-head{display:flex;align-items:center;justify-content:space-between;gap:12px}.case-result-head h3,.step-head h4{margin:0}.step{border-top:1px solid var(--line);padding:18px 0}.expected-observed{display:grid;grid-template-columns:1fr 1fr;gap:14px}.expected-observed h5{margin:8px 0}.command summary code{color:#344054}.exit-code{float:right;color:var(--muted);font-size:11px}.assertion-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:10px}.assertion{border:1px solid var(--line);border-radius:8px;padding:10px}.attachment-image{display:block;max-width:100%;max-height:700px;border:1px solid var(--line);border-radius:8px}.attachment-meta{font-size:12px;color:var(--muted)}.download{display:inline-block;padding:8px 12px;background:var(--brand);color:white;border-radius:7px;text-decoration:none}.reason{background:var(--fail-bg);border:1px solid #fecdca;padding:12px;border-radius:9px}.remarks{background:#f9fafb;padding:12px;border-radius:8px}.hidden{display:none!important}.toolbar{display:flex;gap:8px;justify-content:flex-end;margin:12px 0}.button{border:1px solid var(--line);background:#fff;border-radius:8px;padding:7px 10px;cursor:pointer}.button:hover{background:#f8fafc}.source-json{max-height:500px}@media(max-width:900px){.layout{display:block}.sidebar{position:relative;height:auto}.main{padding:20px}.summary-grid{grid-template-columns:repeat(2,1fr)}.expected-observed{grid-template-columns:1fr}.case-card,.section{margin-left:0}}@media print{.sidebar,.toolbar{display:none}.layout{display:block}.main{max-width:none;padding:0}.case-card{break-inside:avoid}details>*{display:block!important}} +""" + +JS = r""" +(()=>{const q=s=>document.querySelector(s),qa=s=>[...document.querySelectorAll(s)];let filter='ALL';function apply(){const term=(q('#search').value||'').toLowerCase();qa('.case-card').forEach(c=>{const okStatus=filter==='ALL'||c.dataset.status===filter;const okText=!term||c.textContent.toLowerCase().includes(term);c.classList.toggle('hidden',!(okStatus&&okText));});qa('.toc-case').forEach(a=>{const target=q(a.getAttribute('href'));a.classList.toggle('hidden',!target||target.classList.contains('hidden'));});}q('#search').addEventListener('input',apply);qa('.filter').forEach(b=>b.addEventListener('click',()=>{qa('.filter').forEach(x=>x.classList.remove('active'));b.classList.add('active');filter=b.dataset.status;apply();}));q('#expand-all').addEventListener('click',()=>qa('details').forEach(d=>d.open=true));q('#collapse-all').addEventListener('click',()=>qa('details').forEach(d=>d.open=false));})(); +""" + + +def render_report( + plan: Plan, run: dict[str, Any], results: dict[str, tuple[dict[str, Any], Path]] +) -> str: + title = str(plan.index.get("title", plan.index["id"])) + run_id = str(run["id"]) + summary = run["summary"]["by_status"] + body: list[str] = [ + f'

      {html.escape(title)}

      Run {html.escape(run_id)} · {html.escape(str(run.get("started_at", "")))} → {html.escape(str(run.get("finished_at", "")))}

      ' + ] + body.append('

      Run summary

      ') + for status in STATUS: + body.append( + f'
      {status_badge(status)}{summary[status]["count"]}test cases
      ' + ) + body.append( + '
      Software under test' + + json_block(run.get("software_under_test", {})) + + '
      Environment and executor' + + json_block( + { + "environment": run.get("environment", {}), + "executor": run.get("executor", {}), + } + ) + + "
      " + ) + body.append( + '

      Test guide

      ' + + markdown_to_html(plan.guide_path.read_text(encoding="utf-8")) + + "
      " + ) + current_chapter = current_section = None + for case in plan.cases: + if case.chapter_id != current_chapter: + if current_section is not None: + body.append("
") + if current_chapter is not None: + body.append("") + body.append( + f'

{html.escape(case.chapter_title)}

' + ) + if doc := plan.chapter_docs.get(case.chapter_id): + body.append(markdown_to_html(doc.read_text(encoding="utf-8"))) + current_chapter = case.chapter_id + current_section = None + if case.section_id != current_section: + if current_section is not None: + body.append("
") + body.append( + f'

{html.escape(case.section_title)}

' + ) + if doc := plan.section_docs.get(case.section_id): + body.append(markdown_to_html(doc.read_text(encoding="utf-8"))) + current_section = case.section_id + result, result_path = results[case.id] + status = str(result["status"]) + body.append( + f'
{html.escape(case.priority)}

{html.escape(case.id)} · {html.escape(case.title)}

{status_badge(status)}
' + ) + chips = case.requirements + case.risks + case.tags + if chips: + body.append( + '

' + + " ".join(f"{html.escape(x)}" for x in chips) + + "

" + ) + body.append( + markdown_to_html(case.spec_path.read_text(encoding="utf-8")) + "
" + ) + body.append(render_case_result(case, result, result_path, plan) + "
") + if current_section is not None: + body.append("
") + if current_chapter is not None: + body.append("") + body.append( + '

Raw run JSON

Show source
'
+        + html.escape(json.dumps(run, ensure_ascii=False, indent=2))
+        + "
" + ) + nav = nav_html(plan, results) + filters = ( + '
' + + "".join( + f'' + for s in ("ALL",) + STATUS + ) + + "
" + ) + return f"""{html.escape(title)} · {html.escape(run_id)}
{"".join(body)}
""" + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--plan", + type=Path, + required=True, + help="test-plan directory containing index.json", + ) + parser.add_argument("--run-id", required=True, help="run ID under plan/results") + parser.add_argument("--output", type=Path, help="self-contained HTML output path") + parser.add_argument( + "--validate-only", action="store_true", help="validate without rendering" + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv or sys.argv[1:]) + if not args.validate_only and args.output is None: + raise ReportError("--output is required unless --validate-only is used") + plan = load_plan(args.plan) + run, results = load_session_results(plan, args.run_id) + if args.validate_only: + print( + json.dumps( + { + "status": "valid", + "plan_id": plan.index["id"], + "run_id": args.run_id, + "cases": len(plan.cases), + }, + ensure_ascii=False, + ) + ) + return 0 + rendered = render_report(plan, run, results) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + print( + json.dumps( + { + "status": "rendered", + "output": str(args.output), + "bytes": len(rendered.encode()), + "cases": len(plan.cases), + }, + ensure_ascii=False, + ) + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ReportError as error: + print( + json.dumps({"status": "error", "message": str(error)}, ensure_ascii=False), + file=sys.stderr, + ) + raise SystemExit(2) diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md new file mode 100644 index 000000000..85df2f069 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md @@ -0,0 +1,20 @@ + +# TC-GW-PP-001: Forward a Proxy v1 address + + +## Objective + +Verify that a PP-enabled backend receives the declared source address. + + +## Steps + + +### Step 1: Query policy + +**Expected result:** port 8443 has `pp=true`. + + +### Step 2: Send request + +**Expected result:** the backend receives `198.51.100.27:45678`. diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/SHA256SUMS b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/SHA256SUMS new file mode 100644 index 000000000..0a95bd054 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/SHA256SUMS @@ -0,0 +1,5 @@ +bd09187e1cb77d4e60ea4772720d03dc9e783a51d8f943b5918d69fdac559b42 artifacts/backend-capture.json +d4bb03792a2c674dcdaac431a8ecacf66f49fb376bd5b268280716e12db88779 prompt.md +1d589f0c1d6f59e21c870a71d453876752b3fe732b6cd3a3aa8a8b3c959ccf97 result.json +deeff3680a68d0730ac41042d3e94b5d4cdcf6423f8c4d8f0e05038744f1650e runner.json +958eafa2fbb4d9f9308d08d3879c44a831fa791196ae8bb620c18472e2e53215 session.jsonl diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/artifacts/backend-capture.json b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/artifacts/backend-capture.json new file mode 100644 index 000000000..033008b13 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/artifacts/backend-capture.json @@ -0,0 +1 @@ +{"source":"198.51.100.27:45678","status":200} diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/prompt.md b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/prompt.md new file mode 100644 index 000000000..6b295becf --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/prompt.md @@ -0,0 +1 @@ +Run the sample case. diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/result.json b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/result.json new file mode 100644 index 000000000..24ff49af1 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/result.json @@ -0,0 +1,14 @@ +{ + "schema_version": "1.0", + "case_id": "tc-gw-pp-001", + "status": "PASS", + "summary": "The Proxy v1 source address reached the backend.", + "steps": [ + {"id": "tc-gw-pp-001-step-01", "status": "PASS", "observed": "Port 8443 had pp=true."}, + {"id": "tc-gw-pp-001-step-02", "status": "PASS", "observed": "Backend observed 198.51.100.27:45678 and returned HTTP 200."} + ], + "artifacts": [ + {"name": "Backend capture", "path": "artifacts/backend-capture.json"} + ], + "remarks": "" +} diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/runner.json b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/runner.json new file mode 100644 index 000000000..6b13c0c68 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/runner.json @@ -0,0 +1,15 @@ +{ + "schema_version": "1.0", + "run_id": "run-demo", + "case_id": "tc-gw-pp-001", + "agent": {"type": "codex", "model": "sample-model"}, + "session": {"format": "codex-jsonl", "path": "session.jsonl", "events": 7}, + "prompt_path": "prompt.md", + "result_path": "result.json", + "started_at": "2026-07-23T08:00:00.000Z", + "finished_at": "2026-07-23T08:00:10.000Z", + "duration_ms": 10000, + "exit_code": 0, + "result_valid": true, + "result_error": null +} diff --git a/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/session.jsonl b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/session.jsonl new file mode 100644 index 000000000..f05ff50e3 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/session.jsonl @@ -0,0 +1,7 @@ +{"type":"thread.started","thread_id":"demo","model":"sample-model"} +{"type":"item.completed","item":{"type":"agent_message","text":"[DSTACK-TEST-STEP-START tc-gw-pp-001-step-01]"}} +{"type":"item.completed","item":{"type":"command_execution","command":"query-policy","aggregated_output":"{\"port\":8443,\"pp\":true}\n","exit_code":0}} +{"type":"item.completed","item":{"type":"agent_message","text":"[DSTACK-TEST-STEP-END tc-gw-pp-001-step-01 PASS]"}} +{"type":"item.completed","item":{"type":"agent_message","text":"[DSTACK-TEST-STEP-START tc-gw-pp-001-step-02]"}} +{"type":"item.completed","item":{"type":"command_execution","command":"send-request","aggregated_output":"HTTP 200 source=198.51.100.27:45678\n","exit_code":0}} +{"type":"item.completed","item":{"type":"agent_message","text":"[DSTACK-TEST-STEP-END tc-gw-pp-001-step-02 PASS]"}} diff --git a/tools/dstack-test/tests/fixtures/sample-plan/README.md b/tools/dstack-test/tests/fixtures/sample-plan/README.md new file mode 100644 index 000000000..babd28aef --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/README.md @@ -0,0 +1,4 @@ + +# Sample test guide + +This fixture demonstrates an offline dstack test report. diff --git a/tools/dstack-test/tests/fixtures/sample-plan/index.json b/tools/dstack-test/tests/fixtures/sample-plan/index.json new file mode 100644 index 000000000..5e1108f89 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/index.json @@ -0,0 +1,38 @@ +{ + "schema_version": "1.0", + "id": "plan-sample", + "title": "Sample Gateway Test Plan", + "guide": {"path": "README.md", "anchor": "sample-test-guide"}, + "chapters": [ + { + "id": "chapter-gateway", + "order": 1, + "title": "Gateway", + "path": "01-gateway", + "sections": [ + { + "id": "section-proxy-protocol", + "order": 1, + "title": "Proxy Protocol", + "path": "01-gateway/01-proxy-protocol", + "cases": [ + { + "id": "tc-gw-pp-001", + "order": 1, + "title": "Forward a Proxy v1 address", + "priority": "P0", + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001", + "spec": { + "path": "01-gateway/01-proxy-protocol/tc-gw-pp-001/case.md", + "anchor": "tc-gw-pp-001" + }, + "requirements": ["req-gw-pp-v1"], + "risks": ["risk-client-address-loss"], + "tags": ["gateway", "e2e"] + } + ] + } + ] + } + ] +} diff --git a/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/SHA256SUMS b/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/SHA256SUMS new file mode 100644 index 000000000..01f43ac95 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/SHA256SUMS @@ -0,0 +1 @@ +31433a80e8e7c0bccb35f138381b8f05a39806458f94d090d71d44cdab39e9da run.json diff --git a/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/run.json b/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/run.json new file mode 100644 index 000000000..f9d2acde2 --- /dev/null +++ b/tools/dstack-test/tests/fixtures/sample-plan/results/run-demo/run.json @@ -0,0 +1,31 @@ +{ + "schema_version": "1.0", + "id": "run-demo", + "anchor": "run-demo", + "plan_id": "plan-sample", + "status": "COMPLETED", + "started_at": "2026-07-23T08:00:00.000Z", + "finished_at": "2026-07-23T08:00:10.000Z", + "executors": [{"type": "codex", "model": "sample-model"}], + "software_under_test": {"git_revision": "demo"}, + "environment": {"level": "INTEGRATION"}, + "summary": { + "total": 1, + "completed": 1, + "by_status": { + "PASS": {"count": 1, "case_refs": ["#result-tc-gw-pp-001"]}, + "FAIL": {"count": 0, "case_refs": []}, + "BLOCKED": {"count": 0, "case_refs": []}, + "NOT_RUN": {"count": 0, "case_refs": []}, + "SKIPPED": {"count": 0, "case_refs": []} + } + }, + "case_results": [ + { + "id": "tc-gw-pp-001", + "anchor": "result-tc-gw-pp-001", + "status": "PASS", + "result_path": "../../01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/result.json" + } + ] +} diff --git a/tools/dstack-test/tests/test_dstack_test.py b/tools/dstack-test/tests/test_dstack_test.py new file mode 100644 index 000000000..faa2a6c6f --- /dev/null +++ b/tools/dstack-test/tests/test_dstack_test.py @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: D100, D101, D102, D103 + +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +import zipfile +from importlib.machinery import SourceFileLoader +from pathlib import Path +from unittest import mock + +HERE = Path(__file__).resolve().parent +TOOL_DIR = HERE.parent +FIXTURE = HERE / "fixtures" / "sample-plan" +CLI = TOOL_DIR / "dstack-test" +sys.path.insert(0, str(TOOL_DIR)) + +import render # noqa: E402 + +spec = importlib.util.spec_from_loader( + "dstack_test", SourceFileLoader("dstack_test", str(CLI)) +) +if spec is None or spec.loader is None: + raise RuntimeError(f"failed to load {CLI}") +dstack_test = importlib.util.module_from_spec(spec) +spec.loader.exec_module(dstack_test) + + +class DstackTestTests(unittest.TestCase): + def copy_fixture(self, root: Path) -> Path: + plan = root / "plan" + shutil.copytree(FIXTURE, plan) + return plan + + def test_run_command_defaults(self) -> None: + args = dstack_test.build_parser().parse_args( + ["run-plan", "--plan", str(FIXTURE)] + ) + self.assertEqual(args.agent, "codex") + self.assertRegex(args.run_id, r"^run-\d{8}T\d{6}Z-[0-9a-f]{6}$") + + def test_codex_model_is_read_from_config(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + config = Path(temporary) / "config.toml" + config.write_text('model = "test-codex-model"\n', encoding="utf-8") + with mock.patch.dict(os.environ, {"CODEX_HOME": temporary}): + self.assertEqual( + dstack_test.resolve_model("codex", None), "test-codex-model" + ) + + def test_orchestrator_can_record_dependency_skip(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + plan_path = self.copy_fixture(Path(temporary)) + plan = render.load_plan(plan_path) + case = plan.cases[0] + value = dstack_test.skip_case( + case, "run-skip", "prerequisite case failed", ["tc-prereq-001"] + ) + self.assertEqual(value["status"], "SKIPPED") + result_dir = case.path / "results" / "run-skip" + result = dstack_test.validate_summary(case, result_dir / "result.json") + self.assertEqual(result["status"], "SKIPPED") + self.assertIn("tc-prereq-001", (result_dir / "session.jsonl").read_text()) + + def test_validate_and_render_fixture(self) -> None: + plan = render.load_plan(FIXTURE) + valid = dstack_test.validate_run(plan, "run-demo") + self.assertEqual(valid["cases"], 1) + run, results = render.load_session_results(plan, "run-demo") + output = render.render_report(plan, run, results) + self.assertIn("Sample Gateway Test Plan", output) + self.assertIn("Complete agent session (7 events)", output) + self.assertIn("198.51.100.27:45678", output) + self.assertIn("session-tc-gw-pp-001-event-2", output) + + def test_dashboard_exposes_historical_status_and_log(self) -> None: + plan = render.load_plan(FIXTURE) + state = dstack_test.dashboard_state(plan, "run-demo") + self.assertEqual(state["cases"][0]["status"], "PASS") + log = dstack_test.dashboard_log(plan, "run-demo", "case:tc-gw-pp-001", 0) + self.assertGreater(log["next_offset"], 0) + self.assertIn("thread.started", log["text"]) + + def test_finalize_rebuilds_run_summary(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + plan_path = self.copy_fixture(Path(temporary)) + shutil.rmtree(plan_path / "results" / "run-demo") + plan = render.load_plan(plan_path) + run = dstack_test.finalize_run( + plan, "run-demo", {"environment": {"level": "INTEGRATION"}} + ) + self.assertEqual(run["status"], "COMPLETED") + self.assertEqual(run["summary"]["by_status"]["PASS"]["count"], 1) + self.assertEqual( + dstack_test.validate_run(plan, "run-demo")["status"], "valid" + ) + + def test_invalid_session_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + plan_path = self.copy_fixture(Path(temporary)) + session = ( + plan_path + / "01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-demo/session.jsonl" + ) + session.write_text("not json\n", encoding="utf-8") + with self.assertRaises(dstack_test.DstackTestError): + dstack_test.validate_run(render.load_plan(plan_path), "run-demo") + + def test_run_case_with_fake_codex(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + plan = self.copy_fixture(root) + fake_bin = root / "bin" + fake_bin.mkdir() + fake = fake_bin / "codex" + fake.write_text( + """#!/usr/bin/env python3 +import json, pathlib, re, sys +prompt=sys.argv[-1] +match=re.search(r'atomically write the summary to ([^\\n]+/result\\.json)',prompt) +if not match: raise SystemExit(9) +path=pathlib.Path(match.group(1).strip()) +path.parent.mkdir(parents=True,exist_ok=True) +result={ + 'schema_version':'1.0','case_id':'tc-gw-pp-001','status':'PASS','summary':'fake agent passed', + 'steps':[ + {'id':'tc-gw-pp-001-step-01','status':'PASS','observed':'policy matched'}, + {'id':'tc-gw-pp-001-step-02','status':'PASS','observed':'request matched'}], + 'artifacts':[],'remarks':''} +tmp=path.with_suffix('.tmp'); tmp.write_text(json.dumps(result)); tmp.replace(path) +print(json.dumps({'type':'thread.started','model':'fake-model'})) +print(json.dumps({'type':'item.completed','item':{'type':'agent_message','text':'tc-gw-pp-001-step-01 tc-gw-pp-001-step-02'}})) +""", + encoding="utf-8", + ) + fake.chmod(0o755) + env = os.environ.copy() + env["PATH"] = str(fake_bin) + os.pathsep + env["PATH"] + completed = subprocess.run( + [ + str(CLI), + "run-case", + "--agent", + "codex", + "--plan", + str(plan), + "--case", + "tc-gw-pp-001", + "--run-id", + "run-fake", + "--workdir", + str(root), + "--", + "fake execution", + ], + text=True, + capture_output=True, + env=env, + check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + result_dir = ( + plan / "01-gateway/01-proxy-protocol/tc-gw-pp-001/results/run-fake" + ) + self.assertEqual( + json.loads((result_dir / "result.json").read_text())["status"], "PASS" + ) + runner = json.loads((result_dir / "runner.json").read_text()) + self.assertEqual(runner["agent"]["model"], "fake-model") + self.assertTrue((result_dir / "session.jsonl").is_file()) + + def test_package_zip_is_self_contained(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + plan_path = self.copy_fixture(root) + old_run = plan_path / "results" / "run-old" + old_run.mkdir(parents=True) + (old_run / "must-not-be-packaged.txt").write_text("old") + output = root / "report.zip" + dstack_test.package_plan(render.load_plan(plan_path), "run-demo", output) + self.assertGreater(output.stat().st_size, 0) + with zipfile.ZipFile(output) as archive: + names = archive.namelist() + self.assertIn("plan/results/run-demo/run.json", names) + self.assertFalse(any("run-old" in name for name in names)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/dstack-test/web.py b/tools/dstack-test/web.py new file mode 100644 index 000000000..a796cfbde --- /dev/null +++ b/tools/dstack-test/web.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network +# SPDX-License-Identifier: Apache-2.0 +"""Embedded live dashboard for dstack test runs.""" + +import http.server +import json +import threading +import urllib.parse +from typing import Any, Callable + +HTML = r"""dstack test

dstack test

Connecting…
Plan orchestrator
""" + + +class Dashboard: + """Serve a read-only live run dashboard.""" + + def __init__( + self, + state: Callable[[], dict[str, Any]], + log: Callable[[str, int], dict[str, Any]], + host: str, + port: int, + ): + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, _format: str, *_args: Any) -> None: + pass + + def reply(self, value: Any, status: int = 200) -> None: + data = json.dumps(value, ensure_ascii=False).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Cache-Control", "no-store") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self) -> None: + parsed = urllib.parse.urlparse(self.path) + if parsed.path == "/": + data = HTML.encode() + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + elif parsed.path == "/api/state": + self.reply(state()) + elif parsed.path == "/api/log": + query = urllib.parse.parse_qs(parsed.query) + try: + self.reply( + log( + query.get("agent", [""])[0], + int(query.get("offset", ["0"])[0]), + ) + ) + except Exception as error: # noqa: BLE001 - API boundary + self.reply({"error": str(error)}, 400) + else: + self.send_error(404) + + self.server = http.server.ThreadingHTTPServer((host, port), Handler) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + + @property + def address(self) -> tuple[str, int]: + host, port = self.server.server_address[:2] + return str(host), int(port) + + def start(self) -> None: + self.thread.start() + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join()