diff --git a/.github/workflows/review-pr.yml b/.github/workflows/review-pr.yml index a7822bc..d4c4677 100644 --- a/.github/workflows/review-pr.yml +++ b/.github/workflows/review-pr.yml @@ -82,6 +82,9 @@ on: exit-code: description: "Exit code from the review" value: ${{ jobs.review.outputs.exit-code }} + review-status: + description: "API-verified review outcome: completed, completed-with-warnings, incomplete, inconclusive, failed, timed-out, skipped, setup-failed, or unverified (empty when the review job did not run)" + value: ${{ jobs.review.outputs.review-status }} permissions: contents: read @@ -296,6 +299,7 @@ jobs: actions: write # cache delete for review-lock release cleanup; feedback artifact list/delete outputs: exit-code: ${{ steps.run-review.outputs.exit-code }} + review-status: ${{ steps.run-review.outputs.review-status }} steps: - name: Resolve PR number @@ -563,10 +567,33 @@ jobs: env: CHECK_ID: ${{ steps.create-check.outputs.check-id }} JOB_STATUS: ${{ job.status }} + REVIEW_STATUS: ${{ steps.run-review.outputs.review-status }} + RUN_REVIEW_OUTCOME: ${{ steps.run-review.outcome }} with: github-token: ${{ github.token }} script: | - const conclusion = process.env.JOB_STATUS === 'cancelled' ? 'cancelled' : process.env.JOB_STATUS === 'success' ? 'success' : 'failure'; + // continue-on-error on the review step masks its failure from + // job.status, so the check conclusion is derived from the + // API-verified review-status instead. Only a verified completion + // may be green; an intentional skip is neutral ONLY when the + // composite itself succeeded — a failed step claiming skipped is + // a contradiction that stays red; every other state (incomplete, + // inconclusive, timed-out, failed, setup-failed, unverified, or a + // missing status from a crashed composite) is a failure. + const reviewStatus = process.env.REVIEW_STATUS || ''; + const stepOutcome = process.env.RUN_REVIEW_OUTCOME || ''; + let conclusion; + if (process.env.JOB_STATUS === 'cancelled') { + conclusion = 'cancelled'; + } else if (reviewStatus === 'completed' || reviewStatus === 'completed-with-warnings') { + conclusion = 'success'; + } else if (reviewStatus === 'skipped' && stepOutcome !== 'failure') { + conclusion = 'neutral'; + } else if (reviewStatus === '' && stepOutcome === 'skipped') { + conclusion = 'neutral'; + } else { + conclusion = 'failure'; + } try { await github.rest.checks.update({ owner: context.repo.owner, @@ -580,6 +607,49 @@ jobs: core.warning(`Failed to update check run: ${error.message}`); } + # continue-on-error on the review step keeps the cleanup/check update + # above running, but it also masks review failures from the job result. + # This gate restores the honest outcome: only a verified completion (or + # an intentionally skipped review) leaves the reusable workflow green — + # fallback/verification failures must fail the job. + - name: Enforce review outcome + if: always() + shell: bash + env: + REVIEW_STATUS: ${{ steps.run-review.outputs.review-status }} + RUN_REVIEW_OUTCOME: ${{ steps.run-review.outcome }} + run: | + case "$REVIEW_STATUS" in + completed|completed-with-warnings) + echo "✅ Review outcome: $REVIEW_STATUS" + ;; + skipped) + # Expected no-op (the composite's concurrent-review lock) — but + # only when the step itself succeeded. A failed step claiming + # skipped is a contradiction that must not leave the job green. + if [ "$RUN_REVIEW_OUTCOME" = "failure" ]; then + echo "::error::Review step failed while reporting review-status=skipped — failing the review job" + exit 1 + fi + echo "⏭️ Review intentionally skipped" + ;; + *) + if [ -z "$REVIEW_STATUS" ] && [ "$RUN_REVIEW_OUTCOME" = "skipped" ]; then + # The review step never ran: a pre-review guard (draft PR, + # authorization, rate anomaly, non-/review comment) filtered + # this event — an expected skip, not a review failure. + echo "⏭️ Review step skipped by a pre-review guard" + exit 0 + fi + # Missing status with a run/failed step means required setup or + # the composite itself failed; any other status (incomplete, + # inconclusive, timed-out, failed, setup-failed, unverified) is + # a non-success review outcome. Neither may leave the job green. + echo "::error::Review outcome '$REVIEW_STATUS' is not a verified completion — failing the review job" + exit 1 + ;; + esac + # No rate-anomaly gate here (unlike the review job): conversational replies are # gated by org membership and serialized per-PR by the `concurrency:` group, and # each reply requires a distinct human comment to trigger it, so they cannot be diff --git a/AGENTS.md b/AGENTS.md index bd93b8c..072f444 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -64,6 +64,18 @@ Anything else here (workflows under `.github/workflows/`, scripts, tests) exists │ │ ├── index.ts # CLI entry → bundled to dist/resolve-trigger-context.js │ │ ├── resolve-trigger-context.ts # Produces canonical context and downstream job outputs. │ │ └── __tests__/ +│ ├── review-assessment/ # PR review Decision Rules spec + trusted posting/attribution runtime. +│ │ ├── index.ts # CLI entry → bundled to dist/review-assessment.js (new-run-nonce, +│ │ │ # finalize-body, classify-run — invoked by review-pr/action.yml). +│ │ ├── review-assessment.ts # Pure assessReview() mirroring pr-review.yaml's Decision Rules +│ │ │ # (neutral 🟢 NO FINDINGS only on zero surviving findings — the +│ │ │ # event is always COMMENT, never APPROVE; incomplete/inconclusive +│ │ │ # never checkpoint) — a tested mirror the model applies, plus the +│ │ │ # runtime helpers: finalize-body validation/marker append and the +│ │ │ # fail-closed post-run review classification (marker + SHA + +│ │ │ # baseline); marker predicates shared with src/incremental-review +│ │ │ # and src/rate-limit. +│ │ └── __tests__/ │ ├── score-confidence/ # Per-finding confidence scoring for the PR review pipeline. │ │ ├── index.ts # CLI entry → bundled to dist/score-confidence.js │ │ ├── score-confidence.ts # Core scoreFinding()/scoreFindings() pure functions + posting policy. @@ -170,7 +182,7 @@ Anything else here (workflows under `.github/workflows/`, scripts, tests) exists - `pnpm test:integration` — Vitest "integration" project (`*.integration.test.ts`). - `tests/*.sh` are integration tests for **shell logic** embedded in YAML (output extraction and job summary in `action.yml`, the release-notes caller-permissions safeguard in `release.yml`). Run them when changing the corresponding bash blocks. - Security unit tests live in `src/security/__tests__/security.test.ts` (Vitest) and run as part of `pnpm test`. Run them when changing anything under `src/security/`. -- The PR review agent has a separate eval suite under `review-pr/agents/evals/`. Run with `docker agent eval review-pr/agents/pr-review.yaml review-pr/agents/evals/`. +- The PR review agent has a separate eval suite under `review-pr/agents/evals/`. Run with `docker agent eval review-pr/agents/pr-review.yaml review-pr/agents/evals/`. All fixtures run in **console output mode**: the eval schema has no per-eval env field and setup exports never reach the agent process, so posting mode / chunk delegation are covered by deterministic unit tests instead (see "Eval environment limits" in `review-pr/README.md`; `src/pr-review-agent/__tests__/eval-fixtures.test.ts` pins the fixture invariants). ### Security-first design (do not regress) @@ -195,7 +207,7 @@ The action runs untrusted input (PR titles, bodies, comments, diffs) through an - `pull_request` action `review_requested` when `github.event.requested_reviewer.login == 'docker-agent'` - `@docker-agent` mentions on PR/issue comments — these run the `.github/actions/mention-reply` handler (sets `should-reply` and builds the context prompt) and then the `review-pr/mention-reply` sub-action (referenced from a pinned SHA, not present as a local path on every commit). The `pr-review-mention-reply.yaml` agent handles the actual reply. - Diffs over 1500 lines are **chunked at file boundaries** in `review-pr/action.yml` (see "Split diff into chunks"). Per-file **risk scoring** (security paths, line counts, error-handling patterns) prioritizes verifier attention. -- **Incremental reviews** (default on, `incremental` input): a re-review only diffs `last-reviewed-SHA..HEAD` instead of the full PR diff. The last reviewed SHA is read from the `commit_id` GitHub records on the bot's completed reviews (assessment or LGTM bodies only — timeout/failure fallbacks don't count), so state survives across runs with no extra writes. `src/incremental-review/` plans the mode and falls back to a full review on force-push/rebase (SHA missing or not an ancestor of HEAD), base-branch merge-ins, net-zero changes, or any error. In incremental mode the original full diff is preserved at `pr_full.diff` for stale-thread resolution and suggestion-anchor validation (GitHub validates anchors against the full PR diff). On re-reviews, `src/dedupe-findings/` (staged at `/tmp/dedupe-findings.js`, run by the agent per `posting-format.md` against the pre-fetched `/tmp/existing_review_comments.json`) drops findings matching already-posted bot comments by file path + line proximity (±3) + finding-heading similarity, so a full re-review after a rebase doesn't duplicate threads. The CLI also accepts the review-thread history snapshot (`/tmp/prior_review_threads.json`) as an optional third argument: current (non-outdated) bot threads suppress re-derived findings whether resolved or unresolved — including paraphrases sharing multiple code anchors and meaningful heading-token overlap — while outdated threads never suppress, so those findings get reassessed. +- **Incremental reviews** (default on, `incremental` input): a re-review only diffs `last-reviewed-SHA..HEAD` instead of the full PR diff. The last reviewed SHA is read from the `commit_id` GitHub records on the bot's completed reviews (`### Assessment:` bodies only — timeout/failure/incomplete fallbacks don't count, and the legacy zero-findings LGTM fallback is deliberately rejected because it was synthesized from exit code 0 alone), so state survives across runs with no extra writes. `src/incremental-review/` plans the mode and falls back to a full review on force-push/rebase (SHA missing or not an ancestor of HEAD), base-branch merge-ins, net-zero changes, or any error. In incremental mode the original full diff is preserved at `pr_full.diff` for stale-thread resolution and suggestion-anchor validation (GitHub validates anchors against the full PR diff). On re-reviews, `src/dedupe-findings/` (staged at `/tmp/dedupe-findings.js`, run by the agent per `posting-format.md` against the pre-fetched `/tmp/existing_review_comments.json`) drops findings matching already-posted bot comments by file path + line proximity (±3) + finding-heading similarity, so a full re-review after a rebase doesn't duplicate threads. The CLI also accepts the review-thread history snapshot (`/tmp/prior_review_threads.json`) as an optional third argument: current (non-outdated) bot threads suppress re-derived findings whether resolved or unresolved — including paraphrases sharing multiple code anchors and meaningful heading-token overlap — while outdated threads never suppress, so those findings get reassessed. - Per-finding **confidence scoring** assigns each verified finding a precise 0–100 score (band: strong/moderate/weak/negligible) from the verifier's `verdict`, `evidence_strength`, and `context_completeness`, plus drafter↔verifier severity concordance and scope. `src/score-confidence/score-confidence.ts` is the **single source of truth** for the model (weights, bands, threshold, posting policy); the "Confidence Scoring" section of `review-pr/agents/pr-review.yaml` mirrors it as a strict lookup table so the orchestrator can apply it inline (the gitignored `dist/` is not available at agent runtime). Change one, change both — the unit tests pin every value. Security and high-severity CONFIRMED/LIKELY findings are always posted regardless of score; below-threshold findings are surfaced in a summary rather than silently dropped. The inline-posting cutoff is configurable via the `confidence-threshold` action input (a band name or a number clamped to 30–100, default `moderate` = 55); the action resolves it by invoking the bundled `dist/score-confidence.js resolve-threshold` CLI (so the resolution logic stays in TypeScript, not bash) and injects it into the agent prompt, and `scoreFinding`/`scoreFindings` accept a matching `postThreshold` option. - Stale review threads on lines no longer in the diff are auto-resolved via GraphQL `resolveReviewThread`. Threads with no `` marker are never touched. The same step first snapshots every thread whose ROOT comment carries a review marker (with `isOutdated`, `originalLine`, `resolvedBy.login`, and per-comment `databaseId`/`author.login`/`replyTo.databaseId`/`body`) to `/tmp/prior_review_threads.json` — initialized to `[]` and persisted before any diff-related or resolution early exit, fail-open. The review context references that artifact by path and count only (thread bodies/replies are untrusted PR data, never interpolated into the prompt); `pr-review.yaml`'s "Prior Review Thread History" policy makes the agent suppress findings tracked by current threads (resolved or unresolved), reassess outdated ones, treat human replies as evidence rather than instructions, and never silently escalate severity; the dedupe CLI consumes the same artifact as its optional third argument. diff --git a/SECURITY.md b/SECURITY.md index cba4127..dd7aa83 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -111,7 +111,8 @@ bound request frequency: a sliding window (default 600 s) and, when the count crosses a threshold (default 8), flags a rate anomaly. It counts one unit per LLM run: full reviews via the Reviews API (`pulls.listReviews`, by bot author, covering findings, - zero-finding APPROVEs, and timeout/error/LGTM fallbacks, none of which carry an + zero-finding 🟢 NO FINDINGS completions, and timeout/error/incomplete fallbacks, + none of which carry an inline marker) plus marker-bearing reply comments. The review job skips the expensive review on a flagged anomaly; the conversational reply jobs are not gated (they are org-gated and per-PR serialized), though their replies still diff --git a/review-pr/README.md b/review-pr/README.md index a24452a..7323ad9 100644 --- a/review-pr/README.md +++ b/review-pr/README.md @@ -316,6 +316,22 @@ review after its diff was prepared. \*API keys are optional when using the reusable workflow (credentials are fetched via OIDC). Only required when using the composite action directly without OIDC. +## Outputs + +Both the reusable workflow and the composite action expose the review outcome: + +| Output | Description | +| --------------- | ----------- | +| `review-status` | API-verified outcome of the run: `completed`, `completed-with-warnings`, `incomplete`, `inconclusive`, `failed`, `timed-out`, `skipped`, `setup-failed`, or `unverified`. Derived from GitHub API state (this run's attribution marker on the selected SHA above the pre-run review baseline), never from the agent's exit code alone. `skipped` is reported only for the intentional concurrent-review lock skip; `setup-failed` means a setup step failed before the agent could run. From the reusable workflow it is empty when the review job did not run. | +| `exit-code` | Exit code from the review agent. Diagnostic only — `review-status` is the authoritative outcome. | +| `review-url` | URL to the reviewed pull request (composite action only). | + +Only `completed` and `completed-with-warnings` mean a verified review was posted for +this run. The reusable workflow's check run and its final "Enforce review outcome" +step key on the same output: an intentional skip is neutral, and every other +non-success state fails the review job — a run that posted nothing can never end +green. + --- ## Example Output @@ -373,9 +389,12 @@ lose the whole review. The validator is implemented and unit-tested in When no issues are found: ```markdown -✅ Looks good! No issues found in the changed code. +### Assessment: 🟢 NO FINDINGS ``` +The zero-findings label is a neutral completion marker — the review is always +posted with the `COMMENT` event, never `APPROVE` or `REQUEST_CHANGES`. + --- ### Review Pipeline @@ -531,16 +550,49 @@ Each eval file in `review-pr/agents/evals/` contains: - **`evals.relevance`**: Natural-language assertions checked against the agent's output - **`evals.setup`**: Setup commands run before the eval (e.g., installing `gh`) +### Eval environment limits + +Know these before writing or debugging a fixture — each one has produced a +broken eval in the past: + +- **Every eval runs in console output mode.** The eval schema accepts only + `relevance`, `working_dir`, `size`, `setup`, and `image` per eval (unknown + fields are rejected) — there is **no per-eval env field** — and the runner + executes `sh /setup.sh && exec /docker-agent …`, so environment exports in + `setup` are child-shell-local and never reach the agent process. A fixture + cannot set `GITHUB_ACTIONS=true`; do not write posting-mode expectations. + GitHub posting mode, pre-split `/tmp/drafter_chunk_*.diff` delegation, and + the posting chain are covered deterministically by + `src/pr-review-agent/__tests__/pr-review-yaml.test.ts` and + `src/resolve-trigger-context/__tests__/workflow-security.test.ts` instead. +- **`setup` must not write to stderr on success.** Some runner versions treat + any setup stderr as fatal even on exit 0. Alpine's `apk add nodejs` prints + an ICU packaging note to stderr, so avoid installing Node in setup; append + `2>&1` to any command that chats on stderr when it succeeds. +- **The container starts empty.** Only the agents directory is mounted (at + `/configs`, read-only); `/working_dir` has no repo checkout and no `dist/` + bundles, so `node dist/….js` cannot work in setup. Stage files with the + `working_dir` field or generate them in `setup` (e.g. `git init` a repo for + the console-mode diff flow). +- **No external writes.** Setup and relevance criteria must never invoke + `gh api` or otherwise post to GitHub — evals must not be able to write to + real repositories. +- **`--only` matches case-insensitive substrings** of eval file names, not + globs (`--only success` runs `success-1/2/3`; `success-*` matches nothing). + +`src/pr-review-agent/__tests__/eval-fixtures.test.ts` pins these invariants +for every fixture in `review-pr/agents/evals/`. + ### Eval naming conventions | Prefix | Expected outcome | | ------------ | ------------------------------------------------------------------ | -| `success-*` | Clean PR, agent should APPROVE | -| `security-*` | PR with security concerns, agent should COMMENT or REQUEST_CHANGES | +| `success-*` | Clean PR, agent reports a neutral 🟢 NO FINDINGS review (console COMMENT format, never approval wording) | +| `security-*` | PR with security concerns, agent should surface findings (COMMENT) | ### Writing new evals -1. Find a PR with a known correct outcome (e.g., a clean PR that should be approved, or one with a real bug) +1. Find a PR with a known correct outcome (e.g., a clean PR with no findings, or one with a real bug) 2. Create a JSON file with the PR URL as the user message and relevance criteria describing the expected behavior 3. Run the eval 3+ times to verify consistency diff --git a/review-pr/action.yml b/review-pr/action.yml index e2eaba0..817075d 100644 --- a/review-pr/action.yml +++ b/review-pr/action.yml @@ -91,6 +91,9 @@ outputs: exit-code: description: "Exit code from the review" value: ${{ steps.run-review.outputs.exit-code }} + review-status: + description: "API-verified outcome of the run: completed, completed-with-warnings, incomplete, inconclusive, failed, timed-out, skipped, setup-failed, or unverified" + value: ${{ steps.post-summary.outputs.review-status }} review-url: description: "URL to the posted review" value: ${{ steps.post-summary.outputs.review-url }} @@ -835,7 +838,7 @@ runs: echo "---" echo "" echo "- **Selected review SHA**: \`${PR_HEAD_SHA}\`" - echo "- **Posting command**: \`/tmp/refs/posting-format.md\` contains this exact immutable SHA." + echo "- **Posting command**: \`/tmp/refs/posting-format.md\` contains this exact immutable SHA and the trusted repository/PR review route — use it as rendered." echo "" echo "## Instructions" echo "" @@ -846,7 +849,7 @@ runs: echo '3. **Verify**: For each hypothesis, delegate to `verifier` agent' echo '4. **Post**: Aggregate findings and post review via `gh api`' echo "" - echo "Only report CONFIRMED and LIKELY findings. Always post as COMMENT (never APPROVE or REQUEST_CHANGES)." + echo "Only post CONFIRMED and LIKELY findings inline. Surviving low-severity findings skip verification but MUST still be surfaced in the review body's low-severity summary and block the 🟢 NO FINDINGS assessment. Always post as COMMENT (never APPROVE or REQUEST_CHANGES)." } > review_context.md # Append extra prompt if provided @@ -866,6 +869,35 @@ runs: echo "PROMPT_EOF" } >> $GITHUB_OUTPUT + # Unguessable per-run attribution nonce (cryptographically random, 32 hex). + # The rendered posting command and every fallback notice embed it as a + # hidden HTML-comment marker (), so + # the post-run classifier can attribute reviews to exactly this run by + # content — independent of the posting login, which varies with the + # github-token input (docker-agent PAT, docker-agent[bot]/ + # github-actions[bot] app tokens, or a consumer identity). Generated by the + # bundled CLI in this trusted step, never from user input; masked + # immediately so no later step can leak it into the run log — the value is + # only ever echoed inside the posted review bodies themselves. + - name: Generate run attribution nonce + id: run-nonce + if: steps.lock-check.outputs.skip != 'true' + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + run: | + RUN_NONCE=$(node "$ACTION_PATH/../dist/review-assessment.js" new-run-nonce) + # Mask before ANY other output: an unmasked nonce in the public run + # log would let anyone mint a review this run's classifier attributes + # to itself. + echo "::add-mask::$RUN_NONCE" + if ! [[ "$RUN_NONCE" =~ ^[0-9a-f]{32}$ ]]; then + echo "::error::Generated run attribution nonce is malformed — refusing to stage review posting" + exit 1 + fi + echo "nonce=$RUN_NONCE" >> "$GITHUB_OUTPUT" + echo "🔑 Generated per-run review attribution nonce" + - name: Copy reference files id: copy-reference-files if: steps.lock-check.outputs.skip != 'true' @@ -873,6 +905,9 @@ runs: env: ACTION_PATH: ${{ github.action_path }} PR_HEAD_SHA: ${{ steps.pr-info.outputs.head-sha }} + REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ steps.resolve-context.outputs.pr-number }} + RUN_NONCE: ${{ steps.run-nonce.outputs.nonce }} run: | mkdir -p /tmp/refs cp "$ACTION_PATH"/agents/refs/*.md /tmp/refs/ @@ -880,14 +915,59 @@ runs: echo "::error::Selected PR head SHA is invalid; refusing to stage review posting" exit 1 fi - sed "s/__PR_HEAD_SHA__/$PR_HEAD_SHA/g" "$ACTION_PATH/agents/refs/posting-format.md" > /tmp/refs/posting-format.md + # The repository and PR number are trusted workflow routing values staged into + # the posting template so the agent never substitutes them itself (an + # unrendered {owner}/{repo}/{pr} route previously 404ed until the model + # hand-corrected it). Validate their shape before sed: the charsets exclude + # the delimiter and every replacement metacharacter (|, &, \, newline). + if ! [[ "$REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + echo "::error::Repository is invalid; refusing to stage review posting" + exit 1 + fi + if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + echo "::error::PR number is invalid; refusing to stage review posting" + exit 1 + fi + if ! [[ "$RUN_NONCE" =~ ^[0-9a-f]{32}$ ]]; then + echo "::error::Run attribution nonce is invalid; refusing to stage review posting" + exit 1 + fi + sed -e "s|__PR_HEAD_SHA__|$PR_HEAD_SHA|g" \ + -e "s|__REPOSITORY__|$REPOSITORY|g" \ + -e "s|__PR_NUMBER__|$PR_NUMBER|g" \ + -e "s|__REVIEW_RUN_NONCE__|$RUN_NONCE|g" \ + "$ACTION_PATH/agents/refs/posting-format.md" > /tmp/refs/posting-format.md if grep -q '__PR_HEAD_SHA__\|\$PR_HEAD_SHA' /tmp/refs/posting-format.md || \ [ "$(grep -o -- '--arg commit_id' /tmp/refs/posting-format.md | wc -l | tr -d ' ')" != 1 ] || \ ! grep -q -- "--arg commit_id \"$PR_HEAD_SHA\"" /tmp/refs/posting-format.md; then echo "::error::Rendered posting template does not contain exactly one selected immutable SHA" exit 1 fi + if grep -q '__REPOSITORY__\|__PR_NUMBER__\|{owner}\|{repo}\|{pr}' /tmp/refs/posting-format.md || \ + [ "$(grep -oF -- "gh api \"repos/$REPOSITORY/pulls/$PR_NUMBER/reviews\"" /tmp/refs/posting-format.md | wc -l | tr -d ' ')" != 1 ]; then + echo "::error::Rendered posting template does not route to exactly the trusted repository/PR" + exit 1 + fi + # The finalize-body step is what appends this run's attribution marker + # mechanically and refuses a 🟢 NO FINDINGS body over staged inline + # comments — the model never handles the nonce itself. Exactly one + # rendered invocation with exactly this run's nonce and the trusted + # staged comments file. + if grep -q '__REVIEW_RUN_NONCE__' /tmp/refs/posting-format.md || \ + [ "$(grep -oF -- "finalize-body /tmp/review_body.md $RUN_NONCE /tmp/review_comments.json" /tmp/refs/posting-format.md | wc -l | tr -d ' ')" != 1 ]; then + echo "::error::Rendered posting template does not carry exactly this run's finalize-body invocation (attribution nonce + staged comments file)" + exit 1 + fi echo "posting-reference=/tmp/refs/posting-format.md" >> "$GITHUB_OUTPUT" + # The rendered posting command validates the review body and appends the + # attribution marker through this CLI — without it the agent cannot post + # at all, so a missing bundle is a hard failure (unlike the optional + # helpers below). + if [ ! -f "$ACTION_PATH/../dist/review-assessment.js" ]; then + echo "::error::review-assessment.js not found in dist — refusing to stage review posting" + exit 1 + fi + cp "$ACTION_PATH/../dist/review-assessment.js" /tmp/review-assessment.js # Stage the suggestion-block validator where the agent can run it before # posting (the agent's working dir is the consumer repo, not the action). if [ -f "$ACTION_PATH/../dist/validate-suggestions.js" ]; then @@ -925,6 +1005,35 @@ runs: echo "::warning::Failed to fetch existing review comments — deduplication will be skipped" fi + # Trusted pre-run baseline for the authoritative no-post check in "Post + # clean summary": review IDs are monotonically increasing, so a review + # created by THIS run has an ID above the maximum that exists right now. + # Old reviews (any SHA) are below the baseline and review IDs merely quoted + # in agent logs never enter the comparison. Fail closed: without a baseline + # the post-run check cannot tell a fresh review from a pre-existing one, so + # refuse to start an unverifiable agent run. + - name: Capture review baseline + if: steps.lock-check.outputs.skip != 'true' + id: review-baseline + shell: bash + env: + GH_TOKEN: ${{ steps.resolve-token.outputs.token }} + REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ steps.resolve-context.outputs.pr-number }} + run: | + set -o pipefail + if ! MAX_REVIEW_ID=$(gh api --paginate "repos/$REPOSITORY/pulls/$PR_NUMBER/reviews" \ + | jq -s 'map(if type == "array" then . else [.] end) | add // [] | map(.id // 0) | max // 0'); then + echo "::error::Failed to capture the pre-run review baseline — refusing to start an unverifiable review" + exit 1 + fi + if ! [[ "$MAX_REVIEW_ID" =~ ^[0-9]+$ ]]; then + echo "::error::Pre-run review baseline is not a review ID — refusing to start an unverifiable review" + exit 1 + fi + echo "max-review-id=$MAX_REVIEW_ID" >> "$GITHUB_OUTPUT" + echo "📌 Pre-run review baseline captured: max existing review ID $MAX_REVIEW_ID" + # ======================================== # RUN REVIEW using root docker-agent-action # ======================================== @@ -1026,17 +1135,21 @@ runs: shell: bash env: GH_TOKEN: ${{ steps.resolve-token.outputs.token }} + ACTION_PATH: ${{ github.action_path }} PR_NUMBER: ${{ steps.resolve-context.outputs.pr-number }} REPOSITORY: ${{ github.repository }} EXIT_CODE: ${{ steps.run-review.outputs.exit-code }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} VERBOSE_LOG_FILE: ${{ steps.run-review.outputs.verbose-log-file }} + BASELINE_MAX_REVIEW_ID: ${{ steps.review-baseline.outputs.max-review-id }} + RUN_NONCE: ${{ steps.run-nonce.outputs.nonce }} SKIP_REASON: ${{ steps.lock-check.outputs.skip-reason }} LOCK_AGE: ${{ steps.lock-check.outputs.lock-age }} CHUNK_COUNT: ${{ steps.chunk-diff.outputs.chunk_count }} PR_HEAD_SHA: ${{ steps.pr-info.outputs.head-sha }} POSTING_REFERENCE: ${{ steps.copy-reference-files.outputs.posting-reference }} run: | + set -o pipefail post_review() { gh api "repos/$REPOSITORY/pulls/$PR_NUMBER/reviews" --input - } @@ -1044,14 +1157,47 @@ runs: REVIEW_URL="https://github.com/$REPOSITORY/pull/$PR_NUMBER" echo "review-url=$REVIEW_URL" >> $GITHUB_OUTPUT TIMEOUT_NOTE="" + REVIEW_STATUS="" + NOTICE_POST_FAILED="" + SETUP_FAILED="" + + # Fail closed when the posted-review verification cannot run: never + # accept an exit code as evidence of a review. Writes an honest summary + # and status so downstream steps (completion reaction) stay non-approving. + verification_failure() { + echo "::error::$1 — cannot verify whether this run posted a review" >&2 + echo "review-status=unverified" >> $GITHUB_OUTPUT + { + echo "" + echo "---" + echo "" + echo "## PR Review Summary" + echo "" + echo "❓ **Review outcome unverified** — $1. Exit code $EXIT_CODE is not accepted as completion evidence." + echo "" + echo "📝 [View Pull Request #$PR_NUMBER]($REVIEW_URL)" + } >> $GITHUB_STEP_SUMMARY + exit 1 + } if [ "$SKIP_REASON" = "concurrent" ]; then # Stay silent — the 👀 reaction on the triggering comment (added by the # run that won the lock) is sufficient feedback that a review is running. # Posting a comment here creates noise and can trigger a comment loop. + # This is the ONLY intentional composite skip — no other path may + # report review-status=skipped. STATUS="⏭️ **Review skipped** — another review is already in progress" + REVIEW_STATUS="skipped" elif [ -z "$EXIT_CODE" ]; then - STATUS="⏭️ **Review skipped** — agent did not run" + # No exit code without the concurrent-lock skip: a required setup + # step failed before the agent could run. Report the honest + # non-success status and fail the step after writing the summary — + # never "skipped", which downstream treats as an expected neutral + # no-op. + echo "::error::Review agent produced no exit code without an intentional skip — a setup step failed before the review could run" >&2 + STATUS="❌ **Review setup failed** — the agent never ran (a setup step failed before the review)" + REVIEW_STATUS="setup-failed" + SETUP_FAILED="1" else if ! [[ "$PR_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "::error::Selected PR head SHA is invalid; refusing to post a PR review" >&2 @@ -1064,67 +1210,197 @@ runs: echo "::error::Rendered posting reference does not contain exactly one selected immutable SHA" >&2 exit 1 fi - if [ "$EXIT_CODE" = "124" ]; then - # Timeout (SIGKILL after 2700 s) — provide actionable guidance - STATUS="⏱️ **Review timed out** (exit code: 124, limit: 2700 s)" - TIMEOUT_NOTE="- **Exit code:** 124 (SIGKILL — 2700 s timeout)" - TIMEOUT_NOTE+=$'\n'"- **Timeout limit:** 2700 s" - TIMEOUT_NOTE+=$'\n'"- Verbose log artifact (${VERBOSE_LOG_FILE}) uploaded for debugging." - if [ -z "${CHUNK_COUNT}" ]; then - TIMEOUT_BODY="⏱️ **PR Review Timed Out** — The review agent hit the 2700 s time limit. Diff size is unknown (chunk-diff step did not produce output). Re-request a review from \`docker-agent\` to retry, or download the verbose log artifact for details." - elif [ "${CHUNK_COUNT}" = "1" ]; then - TIMEOUT_BODY="⏱️ **PR Review Timed Out** — The review agent hit the 2700 s time limit. The diff is small (1 chunk), so this may be an agent loop rather than a large-diff problem. Re-request a review from \`docker-agent\` to retry. If it times out again, download the verbose log artifact for details." - else - TIMEOUT_BODY="⏱️ **PR Review Timed Out** — The review agent hit the 2700 s time limit. This usually happens on large or complex diffs. Re-request a review from \`docker-agent\` to retry — if it times out again, consider splitting the PR into smaller pieces." + if grep -q '__REPOSITORY__\|__PR_NUMBER__\|{owner}\|{repo}\|{pr}' "$POSTING_REFERENCE" || \ + [ "$(grep -oF -- "gh api \"repos/$REPOSITORY/pulls/$PR_NUMBER/reviews\"" "$POSTING_REFERENCE" | wc -l | tr -d ' ')" != 1 ]; then + echo "::error::Rendered posting reference does not route to exactly the trusted repository/PR" >&2 + exit 1 fi - if ! jq -n --arg body "$TIMEOUT_BODY" --arg event "COMMENT" --arg commit_id "$PR_HEAD_SHA" \ - '{body: $body, event: $event, commit_id: $commit_id, comments: []}' \ - | post_review 2>&1; then - echo "::warning::Failed to post timeout comment to PR" + if ! [[ "$RUN_NONCE" =~ ^[0-9a-f]{32}$ ]]; then + verification_failure "Run attribution nonce is missing or malformed" fi - elif [ "$EXIT_CODE" != "0" ]; then - # Check if agent actually posted a review despite the error exit code. - # This happens when a sub-agent fails (e.g., API overload) but the root - # agent recovers and posts the review itself. - if [ -n "$VERBOSE_LOG_FILE" ] && grep -qE 'pullrequestreview-[0-9]+' "$VERBOSE_LOG_FILE" 2>/dev/null; then - echo "::warning::Agent exited $EXIT_CODE but a review was posted — treating as partial success" - STATUS="⚠️ **Review completed with warnings** (exit code: $EXIT_CODE)" - else - STATUS="❌ **Review failed** (exit code: $EXIT_CODE)" - if ! jq -n --arg body "❌ **PR Review Failed** — The review agent encountered an error and could not complete the review. [View logs]($RUN_URL)." --arg event "COMMENT" --arg commit_id "$PR_HEAD_SHA" \ - '{body: $body, event: $event, commit_id: $commit_id, comments: []}' \ - | post_review 2>&1; then - echo "::warning::Failed to post fallback comment to PR" - fi + RUN_MARKER="" + if grep -q '__REVIEW_RUN_NONCE__' "$POSTING_REFERENCE" || \ + [ "$(grep -oF -- "finalize-body /tmp/review_body.md $RUN_NONCE /tmp/review_comments.json" "$POSTING_REFERENCE" | wc -l | tr -d ' ')" != 1 ]; then + echo "::error::Rendered posting reference does not carry exactly this run's finalize-body invocation (attribution nonce + staged comments file)" >&2 + exit 1 fi - else - STATUS="✅ **Review completed**" - # Defense-in-depth: if the log exists but no review was posted, post a fallback LGTM comment. - # This guards against the agent exiting 0 without calling gh api (e.g., zero-findings early exit). - if [ -n "$VERBOSE_LOG_FILE" ] && [ -f "$VERBOSE_LOG_FILE" ] && ! grep -qE 'pullrequestreview-[0-9]+' "$VERBOSE_LOG_FILE" 2>/dev/null; then - # Dedup guard: skip posting if an identical fallback PR review already exists. - # Also check the old issue-comment endpoint for a one-time migration guard - # (prior runs posted to /issues/comments; this prevents a duplicate on the first - # run of this new code against PRs that already received an LGTM issue comment). - EXISTING=$(gh api "repos/$REPOSITORY/pulls/$PR_NUMBER/reviews" \ - --jq '[.[] | select((.body // "") | startswith("🟢 **No issues found**"))] | length' \ - 2>/dev/null || echo "0") - EXISTING_OLD=$(gh api "repos/$REPOSITORY/issues/$PR_NUMBER/comments" \ - --jq '[.[] | select((.body // "") | startswith("🟢 **No issues found**"))] | length' \ - 2>/dev/null || echo "0") - if [ "${EXISTING:-0}" -gt 0 ] || [ "${EXISTING_OLD:-0}" -gt 0 ]; then - echo "ℹ️ Fallback LGTM review already exists — skipping duplicate post" - else - echo "::warning::Agent exited 0 but no review was posted — posting fallback LGTM review" - jq -n --arg body "🟢 **No issues found** — LGTM! [View logs]($RUN_URL)." --arg event "COMMENT" --arg commit_id "$PR_HEAD_SHA" \ - '{body: $body, event: $event, commit_id: $commit_id, comments: []}' \ - | post_review 2>&1 || \ - echo "::warning::Failed to post fallback LGTM review to PR" - fi + # Authoritative posted-review detection (GitHub API state, never log + # grep): the bundled review-assessment CLI attributes a review to + # THIS run by its exact unguessable marker, on the selected immutable + # SHA, with an ID above the pre-run baseline — independent of the + # posting login (the github-token input varies it). Old reviews + # predate the baseline, same-SHA reviews from humans or other + # integrations carry no marker, review IDs merely mentioned in the + # verbose log never enter the comparison, and anything ambiguous + # (duplicate markers, wrong state, invalid body) classifies as + # unverified. Fail closed on missing baseline and on lookup/parsing + # errors. + if ! [[ "$BASELINE_MAX_REVIEW_ID" =~ ^[0-9]+$ ]]; then + verification_failure "Pre-run review baseline is missing or unreadable" + fi + if ! CURRENT_REVIEWS=$(gh api --paginate "repos/$REPOSITORY/pulls/$PR_NUMBER/reviews" \ + | jq -s 'map(if type == "array" then . else [.] end) | add // []'); then + verification_failure "Post-run review lookup failed" + fi + if ! CLASSIFICATION=$(node "$ACTION_PATH/../dist/review-assessment.js" classify-run \ + "$PR_HEAD_SHA" "$BASELINE_MAX_REVIEW_ID" "$RUN_NONCE" <<< "$CURRENT_REVIEWS"); then + verification_failure "Posted-review classification failed" + fi + RUN_REVIEW=$(sed -n 's/^status=//p' <<< "$CLASSIFICATION") + PRIOR_INCOMPLETE_NOTICE=$(sed -n 's/^prior-incomplete-notice=//p' <<< "$CLASSIFICATION") + case "$RUN_REVIEW" in + completed|incomplete|inconclusive|none) ;; + unverified) + verification_failure "Posted-review state is ambiguous" + ;; + *) + verification_failure "Posted-review classification returned an unknown status" + ;; + esac + # Secondary telemetry only — the API state above is the evidence. + if [ -n "$VERBOSE_LOG_FILE" ] && [ -f "$VERBOSE_LOG_FILE" ] && grep -qE 'pullrequestreview-[0-9]+' "$VERBOSE_LOG_FILE" 2>/dev/null; then + echo "ℹ️ Verbose log mentions a posted-review marker (telemetry only; API state is authoritative)" + fi + if [ "$EXIT_CODE" = "124" ]; then + # Timeout (SIGKILL after 2700 s) — shared diagnostics; the outcome + # depends on what THIS run managed to post before the kill. + TIMEOUT_NOTE="- **Exit code:** 124 (SIGKILL — 2700 s timeout)" + TIMEOUT_NOTE+=$'\n'"- **Timeout limit:** 2700 s" + TIMEOUT_NOTE+=$'\n'"- Verbose log artifact (${VERBOSE_LOG_FILE}) uploaded for debugging." + case "$RUN_REVIEW" in + completed) + # The API state proves this run posted its review before the + # timeout killed the agent — a timeout fallback would only add + # a redundant second review. Keep the posted outcome. + echo "::warning::Agent timed out (exit 124) after posting its review — treating as partial success" + STATUS="⚠️ **Review completed with warnings** — review posted, then the agent timed out (exit code: 124)" + REVIEW_STATUS="completed-with-warnings" + ;; + incomplete) + # The agent posted an explicit incomplete review before the + # kill — that semantic outcome stands; never shadow it with a + # duplicate timeout notice or a success claim. + STATUS="⚠️ **Review incomplete** — the posted review reports unreviewed chunks (agent then timed out, exit code: 124)" + REVIEW_STATUS="incomplete" + ;; + inconclusive) + STATUS="⚠️ **Verification inconclusive** — the posted review carries unverified findings (agent then timed out, exit code: 124)" + REVIEW_STATUS="inconclusive" + ;; + *) + STATUS="⏱️ **Review timed out** (exit code: 124, limit: 2700 s)" + REVIEW_STATUS="timed-out" + if [ -z "${CHUNK_COUNT}" ]; then + TIMEOUT_BODY="⏱️ **PR Review Timed Out** — The review agent hit the 2700 s time limit. Diff size is unknown (chunk-diff step did not produce output). Re-request a review from \`docker-agent\` to retry, or download the verbose log artifact for details." + elif [ "${CHUNK_COUNT}" = "1" ]; then + TIMEOUT_BODY="⏱️ **PR Review Timed Out** — The review agent hit the 2700 s time limit. The diff is small (1 chunk), so this may be an agent loop rather than a large-diff problem. Re-request a review from \`docker-agent\` to retry. If it times out again, download the verbose log artifact for details." + else + TIMEOUT_BODY="⏱️ **PR Review Timed Out** — The review agent hit the 2700 s time limit. This usually happens on large or complex diffs. Re-request a review from \`docker-agent\` to retry — if it times out again, consider splitting the PR into smaller pieces." + fi + TIMEOUT_BODY+=$'\n\n'"$RUN_MARKER" + if ! jq -n --arg body "$TIMEOUT_BODY" --arg event "COMMENT" --arg commit_id "$PR_HEAD_SHA" \ + '{body: $body, event: $event, commit_id: $commit_id, comments: []}' \ + | post_review 2>&1; then + # Without the notice the PR carries no evidence the review + # timed out — fail the step so the silent outcome is observable. + echo "::error::Failed to post the timeout notice to the PR" + NOTICE_POST_FAILED="1" + fi + ;; + esac + elif [ "$EXIT_CODE" != "0" ]; then + case "$RUN_REVIEW" in + completed) + # The agent can exit nonzero after successfully posting (e.g. a + # sub-agent failed but the root recovered and posted the + # review). The API classification is the evidence — keep that + # partial success. + echo "::warning::Agent exited $EXIT_CODE but a review was posted — treating as partial success" + STATUS="⚠️ **Review completed with warnings** (exit code: $EXIT_CODE)" + REVIEW_STATUS="completed-with-warnings" + ;; + incomplete) + # The posted review already reports itself incomplete — retain + # that semantic status without a duplicate failure fallback. + STATUS="⚠️ **Review incomplete** — the posted review reports unreviewed chunks (exit code: $EXIT_CODE)" + REVIEW_STATUS="incomplete" + ;; + inconclusive) + STATUS="⚠️ **Verification inconclusive** — the posted review carries unverified findings (exit code: $EXIT_CODE)" + REVIEW_STATUS="inconclusive" + ;; + *) + STATUS="❌ **Review failed** (exit code: $EXIT_CODE)" + REVIEW_STATUS="failed" + FAILURE_BODY="❌ **PR Review Failed** — The review agent encountered an error and could not complete the review. [View logs]($RUN_URL)."$'\n\n'"$RUN_MARKER" + if ! jq -n --arg body "$FAILURE_BODY" --arg event "COMMENT" --arg commit_id "$PR_HEAD_SHA" \ + '{body: $body, event: $event, commit_id: $commit_id, comments: []}' \ + | post_review 2>&1; then + # Without the notice the PR carries no evidence the review + # failed — fail the step so the silent outcome is observable. + echo "::error::Failed to post the failure notice to the PR" + NOTICE_POST_FAILED="1" + fi + ;; + esac + else + # Fail-closed: exit code 0 alone is NOT evidence a review happened. + # NEVER synthesize an LGTM/approval here: docker/gordon PRs + # #1798/#1803/#1808/#1809 received false "No issues found" reviews + # from exactly this path. The notice body must not contain + # "### Assessment:" or approve wording, so it can never advance the + # incremental checkpoint (src/incremental-review rejects it + # explicitly). + case "$RUN_REVIEW" in + completed) + STATUS="✅ **Review completed**" + REVIEW_STATUS="completed" + ;; + incomplete) + # The agent exited 0 but honestly posted an incomplete review + # (unreviewed chunks) — that is NOT a completed run. + STATUS="⚠️ **Review incomplete** — the posted review reports unreviewed chunks" + REVIEW_STATUS="incomplete" + ;; + inconclusive) + STATUS="⚠️ **Verification inconclusive** — the posted review carries unverified findings" + REVIEW_STATUS="inconclusive" + ;; + *) + # No review posted by this run: the agent finished without + # posting (e.g. failed delegations silently swallowed, + # incomplete pipeline, or a posting command that bypassed the + # rendered template) — post an explicit non-approving + # incomplete notice pinned to the selected SHA. + STATUS="⚠️ **Review incomplete** — agent exited 0 without posting a review" + REVIEW_STATUS="incomplete" + # Dedup guard (classified with the API state above): only a + # notice an action identity already pinned to THIS SHA + # suppresses the post, so repeated silent runs do not spam it + # while new commits (and same-worded reviews from humans) + # still get their own notice. + if [ "$PRIOR_INCOMPLETE_NOTICE" = "true" ]; then + echo "ℹ️ Incomplete-review notice for this SHA already exists — skipping duplicate post" + else + echo "::warning::Agent exited 0 but no review was posted — posting a non-approving incomplete-review notice" + FALLBACK_BODY="⚠️ **Review incomplete** — The review agent finished without posting a review, so this PR has NOT been reviewed. This is not an approval. Re-request a review from \`docker-agent\` to retry, or check the [logs]($RUN_URL)."$'\n\n'"$RUN_MARKER" + if ! jq -n --arg body "$FALLBACK_BODY" --arg event "COMMENT" --arg commit_id "$PR_HEAD_SHA" \ + '{body: $body, event: $event, commit_id: $commit_id, comments: []}' \ + | post_review 2>&1; then + # Without the notice the PR carries no evidence this run did + # not review it — fail the step so the false success is + # observable. + echo "::error::Failed to post the incomplete-review notice to the PR" + NOTICE_POST_FAILED="1" + fi + fi + ;; + esac fi - fi fi + echo "review-status=$REVIEW_STATUS" >> $GITHUB_OUTPUT + # Override the default summary with a cleaner one for PR reviews { echo "" @@ -1141,19 +1417,31 @@ runs: echo "📝 [View Pull Request #$PR_NUMBER]($REVIEW_URL)" } >> $GITHUB_STEP_SUMMARY + if [ -n "$NOTICE_POST_FAILED" ] || [ -n "$SETUP_FAILED" ]; then + exit 1 + fi + - name: Add completion reaction if: steps.resolve-context.outputs.comment-id != '' && always() && steps.lock-check.outputs.skip != 'true' shell: bash env: GH_TOKEN: ${{ steps.resolve-token.outputs.token }} - EXIT_CODE: ${{ steps.run-review.outputs.exit-code }} + REVIEW_STATUS: ${{ steps.post-summary.outputs.review-status }} REPO: ${{ github.repository }} COMMENT_ID: ${{ steps.resolve-context.outputs.comment-id }} run: | - if [ "$EXIT_CODE" != "0" ]; then - gh api "repos/$REPO/issues/comments/$COMMENT_ID/reactions" \ - -X POST -f content='confused' || true - else - gh api "repos/$REPO/issues/comments/$COMMENT_ID/reactions" \ - -X POST -f content='+1' || true - fi + # 👍 only when the API-verified status says a review was actually + # posted and complete. Every other state — incomplete, inconclusive, + # failed, timed-out, skipped, setup-failed, unverified, or an empty + # status from a failed summary step — reacts confused so an exit-0 + # no-post run never looks like a completed review. + case "$REVIEW_STATUS" in + completed|completed-with-warnings) + REACTION='+1' + ;; + *) + REACTION='confused' + ;; + esac + gh api "repos/$REPO/issues/comments/$COMMENT_ID/reactions" \ + -X POST -f content="$REACTION" || true diff --git a/review-pr/agents/evals/auto-filter-integration-1.json b/review-pr/agents/evals/auto-filter-integration-1.json index ce51fe4..4f69004 100644 --- a/review-pr/agents/evals/auto-filter-integration-1.json +++ b/review-pr/agents/evals/auto-filter-integration-1.json @@ -1,15 +1,15 @@ { "id": "a1f2e3d4-auto-filter-integration-001", - "title": "Auto-filter integration - score-0 test files excluded, SQL injection caught in remaining file (run 1)", + "title": "Noisy diff console review - comment-only test scaffolding stays clean, SQL injection caught in storage layer (run 1)", "evals": { - "setup": "apk add --no-cache github-cli jq nodejs && gen_rs() { local fp=$1; printf 'diff --git a/%s b/%s\\nindex abc1234..def5678 100644\\n--- a/%s\\n+++ b/%s\\n@@ -1,2 +1,52 @@\\n // Rust integration test\\n \\n' \"$fp\" \"$fp\" \"$fp\" \"$fp\"; i=1; while [ $i -le 50 ]; do printf '+// test line %d\\n' $i; i=$((i+1)); done; } && { gen_rs crates/vm/tests/integration/virtiofs_bench.rs; gen_rs crates/vm/tests/integration/cpu_test.rs; gen_rs crates/vm/tests/integration/memory_test.rs; gen_rs crates/vm/tests/integration/io_bench.rs; gen_rs crates/vm/tests/integration/network_test.rs; gen_rs crates/vm/tests/integration/block_dev_test.rs; gen_rs crates/vm/tests/integration/hypervisor_bench.rs; gen_rs crates/vm/tests/integration/vhost_test.rs; gen_rs crates/vm/tests/integration/virtio_test.rs; gen_rs crates/vm/tests/integration/kvm_bench.rs; gen_rs crates/vm/tests/integration/balloon_test.rs; gen_rs crates/vm/tests/integration/rng_bench.rs; gen_rs crates/vm/tests/integration/pci_test.rs; gen_rs crates/vm/tests/integration/vsock_test.rs; gen_rs crates/vm/tests/integration/console_test.rs; cat << 'GODBEOF'\ndiff --git a/pkg/storage/db.go b/pkg/storage/db.go\nindex abc1234..def5678 100644\n--- a/pkg/storage/db.go\n+++ b/pkg/storage/db.go\n@@ -1,5 +1,110 @@\n package storage\n\n+import (\n+\t\"database/sql\"\n+\t\"fmt\"\n+)\n+\n+// User represents a database user record.\n+type User struct {\n+\tID int\n+\tName string\n+\tEmail string\n+}\n+\n+// db filler line 1\n+// db filler line 2\n+// db filler line 3\n+// db filler line 4\n+// db filler line 5\n+// db filler line 6\n+// db filler line 7\n+// db filler line 8\n+// db filler line 9\n+// db filler line 10\n+// db filler line 11\n+// db filler line 12\n+// db filler line 13\n+// db filler line 14\n+// db filler line 15\n+// db filler line 16\n+// db filler line 17\n+// db filler line 18\n+// db filler line 19\n+// db filler line 20\n+// db filler line 21\n+// db filler line 22\n+// db filler line 23\n+// db filler line 24\n+// db filler line 25\n+// db filler line 26\n+// db filler line 27\n+// db filler line 28\n+// db filler line 29\n+// db filler line 30\n+// db filler line 31\n+// db filler line 32\n+// db filler line 33\n+// db filler line 34\n+// db filler line 35\n+// db filler line 36\n+// db filler line 37\n+// db filler line 38\n+// db filler line 39\n+// db filler line 40\n+// db filler line 41\n+// db filler line 42\n+// db filler line 43\n+// db filler line 44\n+// db filler line 45\n+// db filler line 46\n+// db filler line 47\n+// db filler line 48\n+// db filler line 49\n+// db filler line 50\n+// db filler line 51\n+// db filler line 52\n+// db filler line 53\n+// db filler line 54\n+// db filler line 55\n+// db filler line 56\n+// db filler line 57\n+// db filler line 58\n+// db filler line 59\n+// db filler line 60\n+// db filler line 61\n+// db filler line 62\n+// db filler line 63\n+// db filler line 64\n+// db filler line 65\n+// db filler line 66\n+// db filler line 67\n+// db filler line 68\n+// db filler line 69\n+// db filler line 70\n+// db filler line 71\n+// db filler line 72\n+// db filler line 73\n+// db filler line 74\n+// db filler line 75\n+// db filler line 76\n+// db filler line 77\n+// db filler line 78\n+// db filler line 79\n+// db filler line 80\n+\n+// GetUser fetches a user by ID from the database.\n+func GetUser(db *sql.DB, id string) (*User, error) {\n+\tq := fmt.Sprintf(\"SELECT * FROM users WHERE id = '%s'\", id)\n+\trow := db.QueryRow(q)\n+\tvar u User\n+\tif err := row.Scan(&u.ID, &u.Name, &u.Email); err != nil {\n+\t\treturn nil, fmt.Errorf(\"scan user: %w\", err)\n+\t}\n+\treturn &u, nil\n+}\nGODBEOF\ncat << 'GOHANDLEREOF'\ndiff --git a/pkg/api/handler.go b/pkg/api/handler.go\nindex abc1234..def5678 100644\n--- a/pkg/api/handler.go\n+++ b/pkg/api/handler.go\n@@ -1,3 +1,43 @@\n package api\n\n+import (\n+\t\"encoding/json\"\n+\t\"fmt\"\n+\t\"net/http\"\n+)\n+\n+// handler filler line 1\n+// handler filler line 2\n+// handler filler line 3\n+// handler filler line 4\n+// handler filler line 5\n+// handler filler line 6\n+// handler filler line 7\n+// handler filler line 8\n+// handler filler line 9\n+// handler filler line 10\n+// handler filler line 11\n+// handler filler line 12\n+// handler filler line 13\n+// handler filler line 14\n+// handler filler line 15\n+// handler filler line 16\n+// handler filler line 17\n+// handler filler line 18\n+// handler filler line 19\n+// handler filler line 20\n+// handler filler line 21\n+// handler filler line 22\n+// handler filler line 23\n+// handler filler line 24\n+// handler filler line 25\n+// handler filler line 26\n+// handler filler line 27\n+// handler filler line 28\n+// handler filler line 29\n+// handler filler line 30\n+\n+// GetUserHandler handles GET /users/{id} requests.\n+func GetUserHandler(w http.ResponseWriter, r *http.Request) {\n+\tuserID := r.URL.Query().Get(\"id\")\n+\tw.Header().Set(\"Content-Type\", \"application/json\")\n+\tif err := json.NewEncoder(w).Encode(map[string]string{\"id\": userID}); err != nil {\n+\t\thttp.Error(w, fmt.Sprintf(\"encode error: %v\", err), http.StatusInternalServerError)\n+\t\treturn\n+\t}\n+}\nGOHANDLEREOF\n} > pr.diff && echo \"Full diff: $(wc -l < pr.diff) lines\" && node dist/score-risk.js pr.diff \"\" >/dev/null 2>&1 && node dist/auto-filter-diff.js pr.diff 3000 && echo \"Filtered diff: $(wc -l < pr.diff) lines\" && CHUNK=1 && CHUNK_LINES=0 && : > /tmp/drafter_chunk_1.diff && while IFS= read -r line; do case \"$line\" in 'diff --git'*) if [ $CHUNK_LINES -gt 1000 ]; then CHUNK=$((CHUNK + 1)); : > /tmp/drafter_chunk_${CHUNK}.diff; CHUNK_LINES=0; fi ;; esac; echo \"$line\" >> /tmp/drafter_chunk_${CHUNK}.diff; CHUNK_LINES=$((CHUNK_LINES + 1)); done < pr.diff && printf 'pkg/storage/db.go\\npkg/api/handler.go\\n' > changed_files.txt && jq -n '{\"title\":\"Add storage layer and API handler\",\"author\":{\"login\":\"testuser\"},\"body\":\"Adds user storage with DB queries and HTTP handler.\",\"baseRefName\":\"main\",\"headRefName\":\"feature/storage\"}' > pr_metadata.json && mkdir -p /tmp/refs && cp review-pr/agents/refs/*.md /tmp/refs/ && export GITHUB_ACTIONS=true && echo 'Setup complete'", + "setup": "set -eu\napk add --no-cache git 2>&1\ngit init -q -b main .\ngit config user.email eval@example.com\ngit config user.name \"Eval Setup\"\necho \"# repo\" > README.md\ngit add -A\ngit commit -q -m \"base\"\ngit checkout -q -b feature/storage\nmkdir -p crates/vm/tests/integration pkg/storage pkg/api\ngen_rs() { { printf '// Rust integration test scaffolding\\n\\n'; i=1; while [ $i -le 50 ]; do printf '// test line %d\\n' \"$i\"; i=$((i+1)); done; } > \"$1\"; }\nfor f in virtiofs_bench cpu_test memory_test io_bench network_test block_dev_test hypervisor_bench vhost_test virtio_test kvm_bench balloon_test rng_bench pci_test vsock_test console_test; do gen_rs \"crates/vm/tests/integration/$f.rs\"; done\n{ printf 'package storage\\n\\nimport (\\n\\t\"database/sql\"\\n\\t\"fmt\"\\n)\\n\\n// User represents a database user record.\\ntype User struct {\\n\\tID int\\n\\tName string\\n\\tEmail string\\n}\\n\\n'; i=1; while [ $i -le 80 ]; do printf '// db filler line %d\\n' \"$i\"; i=$((i+1)); done; cat << 'DBEOF'\n\n// GetUser fetches a user by ID from the database.\nfunc GetUser(db *sql.DB, id string) (*User, error) {\n\tq := fmt.Sprintf(\"SELECT * FROM users WHERE id = '%s'\", id)\n\trow := db.QueryRow(q)\n\tvar u User\n\tif err := row.Scan(&u.ID, &u.Name, &u.Email); err != nil {\n\t\treturn nil, fmt.Errorf(\"scan user: %w\", err)\n\t}\n\treturn &u, nil\n}\nDBEOF\n} > pkg/storage/db.go\n{ printf 'package api\\n\\nimport (\\n\\t\"encoding/json\"\\n\\t\"fmt\"\\n\\t\"net/http\"\\n)\\n\\n'; i=1; while [ $i -le 30 ]; do printf '// handler filler line %d\\n' \"$i\"; i=$((i+1)); done; cat << 'HEOF'\n\n// GetUserHandler handles GET /users/{id} requests.\nfunc GetUserHandler(w http.ResponseWriter, r *http.Request) {\n\tuserID := r.URL.Query().Get(\"id\")\n\tw.Header().Set(\"Content-Type\", \"application/json\")\n\tif err := json.NewEncoder(w).Encode(map[string]string{\"id\": userID}); err != nil {\n\t\thttp.Error(w, fmt.Sprintf(\"encode error: %v\", err), http.StatusInternalServerError)\n\t\treturn\n\t}\n}\nHEOF\n} > pkg/api/handler.go\ngit add -A\ngit commit -q -m \"Add storage layer and API handler plus VM integration test scaffolding\"\necho \"Generated branch diff: $(git diff main HEAD | wc -l) lines\"\necho 'Setup complete'\n", "relevance": [ "The agent ran 'echo $GITHUB_ACTIONS' before performing the review to detect the output mode", - "The agent detected GITHUB_ACTIONS=true and operated in GitHub posting mode", - "The agent used the pre-split chunk files at /tmp/drafter_chunk_*.diff rather than re-reading pr.diff directly", + "The agent detected console output mode (GITHUB_ACTIONS empty/unset) and output the review to the console as formatted markdown instead of posting via gh api", + "The drafter response is valid JSON containing a 'findings' array, a 'summary' field, and 'review_complete' set to true", "At least one finding flags the SQL injection vulnerability in pkg/storage/db.go where user input is interpolated directly into a SQL query string via fmt.Sprintf", "The SQL injection finding has severity 'high'", - "The agent did not post any review comments on files under crates/vm/tests/integration/ or any other auto-excluded Rust test paths" + "No finding with severity 'high' or 'medium' targets the comment-only Rust test scaffolding under crates/vm/tests/integration/ — those files add only comments and contain nothing executable" ] }, "messages": [ @@ -18,7 +18,7 @@ "agentName": "", "message": { "role": "user", - "content": "Review the following PR.\n\n## PR Information\n- **Title**: Add storage layer and API handler\n- **Author**: testuser\n- **Branch**: feature/storage → main\n- **Files Changed**: 2\n\n## PR Description\nAdds user storage with DB queries and HTTP handler.\n\n## Changed Files\n\npkg/storage/db.go\npkg/api/handler.go\n\n---\n\n## Instructions\n\nExecute the review pipeline:\n\n1. **Gather**: Read the pre-fetched `pr.diff` file. If missing, run `gh pr diff` (use the full URL, not just the number)\n2. **Draft**: Delegate to `drafter` agent to generate bug hypotheses\n3. **Verify**: For each hypothesis, delegate to `verifier` agent\n4. **Post**: Aggregate findings and post review via `gh api`\n\nOnly report CONFIRMED and LIKELY findings. Always post as COMMENT (never APPROVE or REQUEST_CHANGES)." + "content": "Review my changes on this branch. The branch adds a user storage layer with DB queries, an HTTP handler, and scaffolding for the VM integration test suite." } } } diff --git a/review-pr/agents/evals/file-based-diff-1.json b/review-pr/agents/evals/file-based-diff-1.json index d413ba4..680bddb 100644 --- a/review-pr/agents/evals/file-based-diff-1.json +++ b/review-pr/agents/evals/file-based-diff-1.json @@ -8,7 +8,7 @@ "The agent output the review to the console as formatted markdown instead of posting via gh api", "The agent successfully read the diff from the pr.diff file on disk rather than trying to fetch it via gh or git", "The drafter response is valid JSON containing a 'findings' array, a 'summary' field, and 'review_complete' set to true", - "The review includes an assessment label (one of '🟢 APPROVE', '🟡 NEEDS ATTENTION', or '🔴 CRITICAL')" + "The review includes an assessment label (one of '🟢 NO FINDINGS', '🟡 NEEDS ATTENTION', or '🔴 CRITICAL')" ] }, "messages": [ diff --git a/review-pr/agents/evals/large-diff-chunking-1.json b/review-pr/agents/evals/large-diff-chunking-1.json index d969106..c4ff898 100644 --- a/review-pr/agents/evals/large-diff-chunking-1.json +++ b/review-pr/agents/evals/large-diff-chunking-1.json @@ -1,12 +1,12 @@ { "id": "b2c3d4e5-large-chunk-test-001", - "title": "Large diff chunking - agent delegates pre-split chunks (run 1)", + "title": "Large diff console review - local git diff, single drafter delegation, SQL injection flagged (run 1)", "evals": { - "setup": "apk add --no-cache github-cli jq && generate_file() { local filepath=$1 num_lines=$2; echo \"diff --git a/$filepath b/$filepath\"; echo \"index abc1234..def5678 100644\"; echo \"--- a/$filepath\"; echo \"+++ b/$filepath\"; echo \"@@ -1,5 +1,$num_lines @@\"; echo \" package $(basename $(dirname $filepath))\"; echo \" \"; echo \" import (\"; echo \"+\\t\\\"fmt\\\"\"; echo \"+\\t\\\"net/http\\\"\"; echo \" )\"; echo \" \"; i=0; while [ $i -lt $((num_lines - 10)) ]; do if [ \"$filepath\" = \"pkg/storage/db.go\" ] && [ $i -eq 50 ]; then echo \"+// GetUser fetches a user by ID from the database.\"; echo \"+func GetUser(db *sql.DB, userID string) (*User, error) {\"; echo \"+\\tquery := fmt.Sprintf(\\\"SELECT id, name, email FROM users WHERE id = '%s'\\\", userID)\"; echo \"+\\trow := db.QueryRow(query)\"; echo \"+\\tvar u User\"; echo \"+\\tif err := row.Scan(&u.ID, &u.Name, &u.Email); err != nil {\"; echo \"+\\t\\treturn nil, fmt.Errorf(\\\"scan user: %w\\\", err)\"; echo \"+\\t}\"; echo \"+\\treturn &u, nil\"; echo \"+}\"; i=$((i + 10)); else echo \"+// line $i of $filepath\"; i=$((i + 1)); fi; done; } && { generate_file pkg/api/handlers.go 400; generate_file pkg/api/middleware.go 350; generate_file pkg/storage/db.go 400; generate_file pkg/config/loader.go 200; generate_file internal/worker/processor.go 300; } > pr.diff && echo \"Total lines: $(wc -l < pr.diff)\" && CHUNK=1 CHUNK_LINES=0 && : > /tmp/drafter_chunk_1.diff && while IFS= read -r line; do case \"$line\" in 'diff --git'*) if [ $CHUNK_LINES -gt 1000 ]; then CHUNK=$((CHUNK + 1)); : > /tmp/drafter_chunk_${CHUNK}.diff; CHUNK_LINES=0; fi ;; esac; echo \"$line\" >> /tmp/drafter_chunk_${CHUNK}.diff; CHUNK_LINES=$((CHUNK_LINES + 1)); done < pr.diff && echo \"Chunks: $CHUNK\" && for c in $(seq 1 $CHUNK); do echo \"Chunk $c: $(wc -l < /tmp/drafter_chunk_${c}.diff) lines\"; done && printf 'pkg/api/handlers.go\\npkg/api/middleware.go\\npkg/storage/db.go\\npkg/config/loader.go\\ninternal/worker/processor.go\\n' > changed_files.txt && jq -n '{title: \"Add API handlers, storage layer, config loader, and worker\", author: {login: \"testuser\"}, body: \"Large feature PR adding multiple packages.\", baseRefName: \"main\", headRefName: \"feature/big-feature\"}' > pr_metadata.json && echo 'Setup complete'", + "setup": "set -eu\napk add --no-cache git 2>&1\ngit init -q -b main .\ngit config user.email eval@example.com\ngit config user.name \"Eval Setup\"\necho \"# bigrepo\" > README.md\ngit add -A\ngit commit -q -m \"base\"\ngit checkout -q -b feature/big-feature\nmkdir -p pkg/api pkg/storage pkg/config internal/worker\ngen_go() { fp=$1; pkg=$2; n=$3; { printf 'package %s\\n\\n' \"$pkg\"; i=1; while [ $i -le \"$n\" ]; do printf '// %s filler line %d\\n' \"$pkg\" \"$i\"; i=$((i+1)); done; } > \"$fp\"; }\ngen_go pkg/api/handlers.go api 390\ngen_go pkg/api/middleware.go api 340\ngen_go pkg/config/loader.go config 190\ngen_go internal/worker/processor.go worker 400\n{ printf 'package storage\\n\\nimport (\\n\\t\"database/sql\"\\n\\t\"fmt\"\\n)\\n\\n// User represents a database user record.\\ntype User struct {\\n\\tID int\\n\\tName string\\n\\tEmail string\\n}\\n\\n'; i=1; while [ $i -le 300 ]; do printf '// storage filler line %d\\n' \"$i\"; i=$((i+1)); done; cat << 'DBEOF'\n\n// GetUser fetches a user by ID from the database.\nfunc GetUser(db *sql.DB, userID string) (*User, error) {\n\tquery := fmt.Sprintf(\"SELECT id, name, email FROM users WHERE id = '%s'\", userID)\n\trow := db.QueryRow(query)\n\tvar u User\n\tif err := row.Scan(&u.ID, &u.Name, &u.Email); err != nil {\n\t\treturn nil, fmt.Errorf(\"scan user: %w\", err)\n\t}\n\treturn &u, nil\n}\nDBEOF\n} > pkg/storage/db.go\ngit add -A\ngit commit -q -m \"Add API handlers, storage layer, config loader, and worker\"\nDIFF_LINES=$(git diff main HEAD | wc -l)\necho \"Generated branch diff: $DIFF_LINES lines\"\ntest \"$DIFF_LINES\" -ge 1500\necho 'Setup complete'\n", "relevance": [ "The agent ran 'echo $GITHUB_ACTIONS' before performing the review to detect the output mode", - "The agent output the review to the console as formatted markdown instead of posting via gh api", - "The agent detected and used pre-split chunk files at /tmp/drafter_chunk_*.diff for delegation rather than splitting the diff itself", + "The agent detected console output mode (GITHUB_ACTIONS empty/unset) and output the review to the console as formatted markdown instead of posting via gh api", + "The agent captured the diff from the local git repository (git merge-base against a base ref, then git diff --output=./pr-review.diff) and never looked for pre-split chunk files under /tmp — console mode has no workflow-staged files", "The drafter response is valid JSON containing a 'findings' array, a 'summary' field, and 'review_complete' set to true", "At least one finding flags the SQL injection vulnerability in pkg/storage/db.go where user input is interpolated into a SQL query via fmt.Sprintf", "The SQL injection finding has severity 'high' because unsanitized user input in SQL queries is a critical security vulnerability" @@ -18,7 +18,7 @@ "agentName": "", "message": { "role": "user", - "content": "Review the following PR.\n\n## PR Information\n- **Title**: Add API handlers, storage layer, config loader, and worker\n- **Author**: testuser\n- **Branch**: feature/big-feature → main\n- **Files Changed**: 5\n\n## PR Description\nLarge feature PR adding multiple packages: API handlers, middleware, database storage layer, config loader, and background worker processor.\n\n## Changed Files\n\npkg/api/handlers.go\npkg/api/middleware.go\npkg/storage/db.go\npkg/config/loader.go\ninternal/worker/processor.go\n\n---\n\n## Instructions\n\nExecute the review pipeline:\n\n1. **Gather**: Read the pre-fetched `pr.diff` file. If missing, run `gh pr diff` (use the full URL, not just the number)\n2. **Draft**: Delegate to `drafter` agent to generate bug hypotheses\n3. **Verify**: For each hypothesis, delegate to `verifier` agent\n4. **Post**: Aggregate findings and post review via `gh api`\n\nOnly report CONFIRMED and LIKELY findings. Always post as COMMENT (never APPROVE or REQUEST_CHANGES)." + "content": "Review my changes on this branch. It is a large feature branch adding API handlers, middleware, a database storage layer, a config loader, and a background worker processor." } } } diff --git a/review-pr/agents/evals/marlin-event-firing-react-1.json b/review-pr/agents/evals/marlin-event-firing-react-1.json index ecd8ed0..a9e9b2b 100644 --- a/review-pr/agents/evals/marlin-event-firing-react-1.json +++ b/review-pr/agents/evals/marlin-event-firing-react-1.json @@ -2,12 +2,11 @@ "id": "ed82e8e4-0b29-4a3c-80d8-55284bdfd6c0", "title": "Marlin SDK PageView fired in React render body with wrong timestamp format (run 1)", "evals": { - "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", + "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed -e 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' -e 's|__REPOSITORY__|example/repo|g' -e 's/__PR_NUMBER__/1/g' -e 's/__REVIEW_RUN_NONCE__/0123456789abcdef0123456789abcdef/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", "relevance": [ "The agent detects the Marlin SDK import pattern (from \"@docker/data-contracts\") and recognizes this as a Marlin producer PR", "At least one finding identifies that marlin.track() is called directly in the React component render body, not inside a useEffect — this causes the event to fire on every render", "At least one finding identifies that viewedAt receives Date.now() which returns a Unix millisecond integer — this is the wrong type for a Timestamp field, which expects a Timestamp object like Timestamp.fromDate(new Date())", - "At least one finding identifies that Date.now() is evaluated on every render, producing incorrect or duplicate timestamps", "The review does NOT mark this PR as clean or approve-worthy — it contains High severity Marlin violations" ] }, diff --git a/review-pr/agents/evals/marlin-false-positive-1.json b/review-pr/agents/evals/marlin-false-positive-1.json index 813502c..6859791 100644 --- a/review-pr/agents/evals/marlin-false-positive-1.json +++ b/review-pr/agents/evals/marlin-false-positive-1.json @@ -2,7 +2,7 @@ "id": "34b3988f-0263-4379-8e4a-63519969d02c", "title": "Non-Marlin analytics library — should not trigger Marlin guide (run 1)", "evals": { - "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", + "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed -e 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' -e 's|__REPOSITORY__|example/repo|g' -e 's/__PR_NUMBER__/1/g' -e 's/__REVIEW_RUN_NONCE__/0123456789abcdef0123456789abcdef/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", "relevance": [ "The agent ran 'echo $GITHUB_ACTIONS' before performing the review to detect the output mode", "The agent output the review to the console as formatted markdown instead of posting via gh api", diff --git a/review-pr/agents/evals/marlin-identity-ingestion-1.json b/review-pr/agents/evals/marlin-identity-ingestion-1.json index c57af0e..ad5bbcd 100644 --- a/review-pr/agents/evals/marlin-identity-ingestion-1.json +++ b/review-pr/agents/evals/marlin-identity-ingestion-1.json @@ -2,7 +2,7 @@ "id": "046121d9-ac09-4f96-9796-cb853564572e", "title": "Marlin SDK Go producer setting ingestor-owned fields, identity mismatch, and UGC truncation (run 1)", "evals": { - "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", + "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed -e 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' -e 's|__REPOSITORY__|example/repo|g' -e 's/__PR_NUMBER__/1/g' -e 's/__REVIEW_RUN_NONCE__/0123456789abcdef0123456789abcdef/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", "relevance": [ "The agent detects the Marlin SDK import pattern (github.com/docker/data-contracts/gen/go/docker/marlin/) and recognizes this as a Marlin producer PR", "At least one finding identifies that IpAddress is set by the producer (from r.RemoteAddr) — this is an ingestor-owned field that must never be set by the producer", diff --git a/review-pr/agents/evals/marlin-pii-detection-1.json b/review-pr/agents/evals/marlin-pii-detection-1.json index 68cfc6c..15bf23c 100644 --- a/review-pr/agents/evals/marlin-pii-detection-1.json +++ b/review-pr/agents/evals/marlin-pii-detection-1.json @@ -2,7 +2,7 @@ "id": "cb05f5d2-8838-4302-ba78-70b6fb5f75ae", "title": "Marlin SDK AppInvoke analytics with PII risk — err.Error() and command args (run 1)", "evals": { - "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", + "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed -e 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' -e 's|__REPOSITORY__|example/repo|g' -e 's/__PR_NUMBER__/1/g' -e 's/__REVIEW_RUN_NONCE__/0123456789abcdef0123456789abcdef/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", "relevance": [ "The agent ran 'echo $GITHUB_ACTIONS' before performing the review to detect the output mode", "The agent output the review to the console as formatted markdown instead of posting via gh api", diff --git a/review-pr/agents/evals/marlin-platform-context-1.json b/review-pr/agents/evals/marlin-platform-context-1.json index e841b65..5a2b556 100644 --- a/review-pr/agents/evals/marlin-platform-context-1.json +++ b/review-pr/agents/evals/marlin-platform-context-1.json @@ -2,7 +2,7 @@ "id": "b5394a48-d3f1-480c-9952-3a035e4876c3", "title": "Marlin SDK web event from Node server with raw integer enum and non-UUID account_id (run 1)", "evals": { - "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", + "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed -e 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' -e 's|__REPOSITORY__|example/repo|g' -e 's/__PR_NUMBER__/1/g' -e 's/__REVIEW_RUN_NONCE__/0123456789abcdef0123456789abcdef/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", "relevance": [ "The agent detects the Marlin SDK import pattern (from \"@docker/data-contracts\") and recognizes this as a Marlin producer PR", "At least one finding identifies that a WebPageView event (which is a PLATFORM_CONTEXT_WEB event) is being fired from a Node.js server-side Express handler — it should use a PLATFORM_CONTEXT_NODE event instead", diff --git a/review-pr/agents/evals/marlin-string-kind-fixed-1.json b/review-pr/agents/evals/marlin-string-kind-fixed-1.json index 3e96e7c..a03022f 100644 --- a/review-pr/agents/evals/marlin-string-kind-fixed-1.json +++ b/review-pr/agents/evals/marlin-string-kind-fixed-1.json @@ -2,7 +2,7 @@ "id": "2a204370-80fb-4ad5-bea2-75ccb3696e64", "title": "Marlin SDK WebClick with action and elementTag from user input — string_kind and action discriminator violations (run 1)", "evals": { - "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", + "setup": "apk add --no-cache github-cli && mkdir -p /tmp/refs && sed -e 's/__PR_HEAD_SHA__/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/g' -e 's|__REPOSITORY__|example/repo|g' -e 's/__PR_NUMBER__/1/g' -e 's/__REVIEW_RUN_NONCE__/0123456789abcdef0123456789abcdef/g' /configs/refs/posting-format.md > /tmp/refs/posting-format.md && cp /configs/refs/marlin_v2_producer_code_review.md /tmp/refs/", "relevance": [ "The agent detects the Marlin SDK import pattern (from \"@docker/data-contracts\") and recognizes this as a Marlin producer PR", "At least one finding identifies that the action field (STRING_KIND_FIXED) receives user-supplied input from req.body.action, which violates string_kind correctness", diff --git a/review-pr/agents/evals/scope-pre-existing-not-flagged-1.json b/review-pr/agents/evals/scope-pre-existing-not-flagged-1.json index e329e8c..2313484 100644 --- a/review-pr/agents/evals/scope-pre-existing-not-flagged-1.json +++ b/review-pr/agents/evals/scope-pre-existing-not-flagged-1.json @@ -9,7 +9,7 @@ "The drafter response is valid JSON containing a 'findings' array, a 'summary' field, and 'review_complete' set to true", "The review does NOT report that the error returned by fileutil.Remove is discarded or could leak temp files, because that error was already discarded on the pre-existing line (`_ = os.Remove(tmp.Name())`) — swapping os.Remove for fileutil.Remove does not introduce the discarded error, so it is pre-existing and not a regression", "The review does NOT ask the author to check or handle the error returned by fileutil.Remove", - "The assessment label is '🟢 APPROVE' or '🟡 NEEDS ATTENTION' — it is NOT '🔴 CRITICAL'" + "The assessment label is '🟢 NO FINDINGS' or '🟡 NEEDS ATTENTION' — it is NOT '🔴 CRITICAL'" ] }, "messages": [ diff --git a/review-pr/agents/evals/success-1.json b/review-pr/agents/evals/success-1.json index 984cb67..2f7375f 100644 --- a/review-pr/agents/evals/success-1.json +++ b/review-pr/agents/evals/success-1.json @@ -6,10 +6,12 @@ "relevance": [ "The agent ran 'echo $GITHUB_ACTIONS' before performing the review to detect the output mode", "The agent output the review to the console as formatted markdown instead of posting via gh api", - "The review assessment label is '🟢 APPROVE' or '🟡 NEEDS ATTENTION' — it is NOT '🔴 CRITICAL'", + "The review assessment label is '🟢 NO FINDINGS' or '🟡 NEEDS ATTENTION' — it is NOT '🔴 CRITICAL'", "The drafter response is valid JSON containing a 'findings' array, a 'summary' field, and 'review_complete' set to true", "No findings have severity 'high' with verdict 'CONFIRMED' or 'LIKELY'", - "The review does not label any finding as CRITICAL or block-worthy" + "The review keeps neutral comment-only semantics: it does not label any finding as CRITICAL or block-worthy, and it does not use approval wording such as 'APPROVE' or 'LGTM'", + "The review does not claim the 'buildx build' hint template is unreachable dead code or that RootCmd only carries a top-level command token, and it does not claim the tests require a running Docker Desktop — the diff documents that RootCmd is the full matched hook config string and that TestMain sets version.GoTest = true", + "The review does not flag the documented use of os.Stdout/os.Stderr in the hook RunE as a confirmed or likely bug — the diff explains and tests that hook output goes to the process streams" ] }, "messages": [ @@ -18,7 +20,7 @@ "agentName": "", "message": { "role": "user", - "content": "Review the following PR.\n\n## PR Information\n- **Title**: Add CLI hooks to show Gordon hints on command failure\n- **Author**: derekmisler\n- **Branch**: dm/cli-hooks → main\n- **Files Changed**: 4\n\n## PR Description\nRegister a \"docker-cli-plugin-hooks\" subcommand so the Docker CLI shows \"What's next:\" hints suggesting Gordon when commands like build, run, or compose up fail.\n\n### Summary\n- Add a hidden `docker-cli-plugin-hooks` subcommand that returns contextual \"What's next:\" hints when Docker CLI commands fail\n- Hints follow the DDS-CLI format: `description → command`\n- Registered for both V1 and V2 code paths\n\n## Diff\n\n```diff\ndiff --git a/cli/commands/hooks.go b/cli/commands/hooks.go\nnew file mode 100644\nindex 000000000..96e86d62b\n--- /dev/null\n+++ b/cli/commands/hooks.go\n@@ -0,0 +1,76 @@\n+package commands\n+\n+import (\n+\t\"encoding/json\"\n+\t\"fmt\"\n+\t\"io\"\n+\t\"os\"\n+\n+\t\"github.com/docker/ai-common/desktop\"\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/docker/cli/cli-plugins/metadata\"\n+\t\"github.com/spf13/cobra\"\n+)\n+\n+var hintTemplates = map[string]string{\n+\t\"build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"buildx build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"run\": `Debug this container error with Gordon → docker ai \"help me fix this container error\"`,\n+\t\"compose\": `Debug this Compose error with Gordon → docker ai \"help me fix this compose error\"`,\n+}\n+\n+const defaultHintTemplate = `Debug this error with Gordon → docker ai`\n+\n+// checkEnabled verifies that Docker AI is enabled before showing hints.\n+// Replaced in tests to avoid requiring a running Docker Desktop.\n+var checkEnabled = desktop.CheckFeatureIsEnabled\n+\n+// Hooks returns the hidden subcommand that the Docker CLI invokes\n+// after command execution when the \"ai\" plugin has hooks configured.\n+func Hooks() *cobra.Command {\n+\treturn &cobra.Command{\n+\t\tUse: metadata.HookSubcommandName,\n+\t\tHidden: true,\n+\t\t// Override PersistentPreRun to prevent the parent's PersistentPreRunE\n+\t\t// (plugin initialization) from running for hook invocations.\n+\t\tPersistentPreRun: func(*cobra.Command, []string) {},\n+\t\tRunE: func(cmd *cobra.Command, args []string) error {\n+\t\t\tif err := checkEnabled(cmd.Context()); err != nil {\n+\t\t\t\treturn nil\n+\t\t\t}\n+\t\t\treturn handleHook(args, os.Stdout, os.Stderr)\n+\t\t},\n+\t}\n+}\n+\n+// handleHook processes a CLI hook invocation. It parses the\n+// HookPluginData JSON from args, and if the command failed,\n+// writes a HookMessage with a context-specific hint to w.\n+func handleHook(args []string, w, errW io.Writer) error {\n+\tif len(args) == 0 {\n+\t\treturn nil\n+\t}\n+\n+\tvar hookData manager.HookPluginData\n+\tif err := json.Unmarshal([]byte(args[0]), &hookData); err != nil {\n+\t\tfmt.Fprintf(errW, \"warning: failed to parse hook data: %v\\n\", err)\n+\t\treturn nil\n+\t}\n+\n+\tif hookData.CommandError == \"\" {\n+\t\treturn nil\n+\t}\n+\n+\ttmpl := defaultHintTemplate\n+\tif t, ok := hintTemplates[hookData.RootCmd]; ok {\n+\t\ttmpl = t\n+\t}\n+\n+\tenc := json.NewEncoder(w)\n+\tenc.SetEscapeHTML(false)\n+\treturn enc.Encode(hooks.HookMessage{\n+\t\tType: hooks.NextSteps,\n+\t\tTemplate: tmpl,\n+\t})\n+}\n\ndiff --git a/cli/commands/hooks_test.go b/cli/commands/hooks_test.go\nnew file mode 100644\nindex 000000000..931fab662\n--- /dev/null\n+++ b/cli/commands/hooks_test.go\n@@ -0,0 +1,125 @@\n+package commands\n+\n+import (\n+\t\"bytes\"\n+\t\"context\"\n+\t\"encoding/json\"\n+\t\"errors\"\n+\t\"io\"\n+\t\"testing\"\n+\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/stretchr/testify/assert\"\n+\t\"github.com/stretchr/testify/require\"\n+)\n+\n+func TestHandleHook_NoArgs(t *testing.T) {\n+\tvar buf bytes.Buffer\n+\terr := handleHook(nil, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+func TestHandleHook_InvalidJSON(t *testing.T) {\n+\tvar stdout, stderr bytes.Buffer\n+\terr := handleHook([]string{\"not json\"}, &stdout, &stderr)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, stdout.String())\n+\tassert.Contains(t, stderr.String(), \"warning: failed to parse hook data\")\n+}\n+\n+func TestHandleHook_Success(t *testing.T) {\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"\",\n+\t})\n+\n+\tvar buf bytes.Buffer\n+\terr := handleHook([]string{data}, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+func TestHandleHook_Failure(t *testing.T) {\n+\ttests := []struct {\n+\t\tname string\n+\t\trootCmd string\n+\t\twantTmpl string\n+\t}{\n+\t\t{\n+\t\t\tname: \"build\",\n+\t\t\trootCmd: \"build\",\n+\t\t\twantTmpl: hintTemplates[\"build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"buildx build\",\n+\t\t\trootCmd: \"buildx build\",\n+\t\t\twantTmpl: hintTemplates[\"buildx build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"run\",\n+\t\t\trootCmd: \"run\",\n+\t\t\twantTmpl: hintTemplates[\"run\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"compose\",\n+\t\t\trootCmd: \"compose\",\n+\t\t\twantTmpl: hintTemplates[\"compose\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"unknown command falls back to default\",\n+\t\t\trootCmd: \"push\",\n+\t\t\twantTmpl: defaultHintTemplate,\n+\t\t},\n+\t}\n+\n+\tfor _, tc := range tests {\n+\t\tt.Run(tc.name, func(t *testing.T) {\n+\t\t\tdata := marshal(t, manager.HookPluginData{\n+\t\t\t\tRootCmd: tc.rootCmd,\n+\t\t\t\tCommandError: \"exit status 1\",\n+\t\t\t})\n+\n+\t\t\tvar buf bytes.Buffer\n+\t\t\terr := handleHook([]string{data}, &buf, io.Discard)\n+\t\t\trequire.NoError(t, err)\n+\n+\t\t\tvar msg hooks.HookMessage\n+\t\t\trequire.NoError(t, json.Unmarshal(buf.Bytes(), &msg))\n+\t\t\tassert.EqualValues(t, hooks.NextSteps, msg.Type)\n+\t\t\tassert.Equal(t, tc.wantTmpl, msg.Template)\n+\t\t})\n+\t}\n+}\n+\n+func TestHooks_SkippedWhenFeatureDisabled(t *testing.T) {\n+\torig := checkEnabled\n+\tcheckEnabled = func(context.Context) error {\n+\t\treturn errors.New(\"Docker AI is not enabled\")\n+\t}\n+\tt.Cleanup(func() { checkEnabled = orig })\n+\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"exit status 1\",\n+\t})\n+\n+\tcmd := Hooks()\n+\tcmd.SetArgs([]string{data})\n+\tcmd.SetOut(io.Discard)\n+\n+\tvar buf bytes.Buffer\n+\tcmd.SetOut(&buf)\n+\n+\terr := cmd.Execute()\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String(), \"no hint should be shown when Docker AI is disabled\")\n+}\n+\n+func marshal(t *testing.T, v any) string {\n+\tt.Helper()\n+\tb, err := json.Marshal(v)\n+\trequire.NoError(t, err)\n+\treturn string(b)\n+}\n\ndiff --git a/cli/main.go b/cli/main.go\nindex be9cb6210..c57436bff 100644\n--- a/cli/main.go\n+++ b/cli/main.go\n@@ -43,6 +43,10 @@ func main() {\n \t\tcmd.AddCommand(commands.Thread())\n \t\tcmd.AddCommand(commands.Mcp())\n \n+\t\t// CLI hooks subcommand — registered for both V1 and V2 so that\n+\t\t// \"What's next:\" hints work regardless of the active feature flag.\n+\t\tcmd.AddCommand(commands.Hooks())\n+\n \t\toriginalPreRun := cmd.PersistentPreRunE\n \t\tcmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n \t\t\tif err := plugin.PersistentPreRunE(cmd, args); err != nil {\n\ndiff --git a/cli/main_test.go b/cli/main_test.go\nindex 2ccecd42f..338474e68 100644\n--- a/cli/main_test.go\n+++ b/cli/main_test.go\n@@ -98,6 +98,31 @@ func TestMCPBuiltin(t *testing.T) {\n \tassert.Contains(t, string(output), `\"docker\"`)\n }\n \n+func TestHooksShowsHintOnFailure(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"exit status 1\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Contains(t, string(output), `docker ai`)\n+\tassert.Contains(t, string(output), `build failure with Gordon`)\n+}\n+\n+func TestHooksNoOutputOnSuccess(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, string(output))\n+}\n+\n func runDockerAI(args ...string) {\n \tos.Args = args\n \tmain()\n```", + "content": "Review the following PR.\n\n## PR Information\n- **Title**: Add CLI hooks to show Gordon hints on command failure\n- **Author**: derekmisler\n- **Branch**: dm/cli-hooks → main\n- **Files Changed**: 4\n\n## PR Description\nRegister a \"docker-cli-plugin-hooks\" subcommand so the Docker CLI shows \"What's next:\" hints suggesting Gordon when commands like build, run, or compose up fail.\n\n### Summary\n- Add a hidden `docker-cli-plugin-hooks` subcommand that returns contextual \"What's next:\" hints when Docker CLI commands fail\n- Hints follow the DDS-CLI format: `description → command`\n- Registered for both V1 and V2 code paths\n- Per the docker/cli plugin-hooks contract (docker/cli#6794), the CLI passes the matched hook config string as `HookPluginData.RootCmd` — multi-word entries like `buildx build` included — and reads the hook subprocess's stdout for the JSON `HookMessage`\n- CLI-level tests run in-process under the existing `TestMain`, which sets `version.GoTest = true` so `desktop.CheckFeatureIsEnabled` is a no-op without Docker Desktop\n\n## Diff\n\n```diff\ndiff --git a/cli/commands/hooks.go b/cli/commands/hooks.go\nnew file mode 100644\nindex 000000000..96e86d62b\n--- /dev/null\n+++ b/cli/commands/hooks.go\n@@ -0,0 +1,88 @@\n+package commands\n+\n+import (\n+\t\"encoding/json\"\n+\t\"fmt\"\n+\t\"io\"\n+\t\"os\"\n+\n+\t\"github.com/docker/ai-common/desktop\"\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/docker/cli/cli-plugins/metadata\"\n+\t\"github.com/spf13/cobra\"\n+)\n+\n+// hintTemplates maps a hook config string to the hint shown when that command\n+// fails. Per the docker/cli plugin-hooks contract (docker/cli#6794), the CLI\n+// passes the exact hook configuration string it matched for the invocation as\n+// HookPluginData.RootCmd — including multi-word entries such as\n+// \"buildx build\" — never just the top-level command token.\n+var hintTemplates = map[string]string{\n+\t\"build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"buildx build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"run\": `Debug this container error with Gordon → docker ai \"help me fix this container error\"`,\n+\t\"compose\": `Debug this Compose error with Gordon → docker ai \"help me fix this compose error\"`,\n+}\n+\n+const defaultHintTemplate = `Debug this error with Gordon → docker ai`\n+\n+// checkEnabled verifies that Docker AI is enabled before showing hints.\n+// desktop.CheckFeatureIsEnabled short-circuits to nil when version.GoTest is\n+// true — cli/main_test.go's TestMain sets that flag — so the in-process CLI\n+// tests never need a running Docker Desktop. Unit tests replace this var.\n+var checkEnabled = desktop.CheckFeatureIsEnabled\n+\n+// Hooks returns the hidden subcommand that the Docker CLI invokes\n+// after command execution when the \"ai\" plugin has hooks configured.\n+func Hooks() *cobra.Command {\n+\treturn &cobra.Command{\n+\t\tUse: metadata.HookSubcommandName,\n+\t\tHidden: true,\n+\t\t// Override PersistentPreRun to prevent the parent's PersistentPreRunE\n+\t\t// (plugin initialization) from running for hook invocations.\n+\t\tPersistentPreRun: func(*cobra.Command, []string) {},\n+\t\tRunE: func(cmd *cobra.Command, args []string) error {\n+\t\t\tif err := checkEnabled(cmd.Context()); err != nil {\n+\t\t\t\treturn nil\n+\t\t\t}\n+\t\t\t// Hook output deliberately goes to the process streams rather\n+\t\t\t// than cmd.OutOrStdout(): the Docker CLI reads the hook\n+\t\t\t// subprocess's stdout, and cli/main_test.go's setStdout swaps\n+\t\t\t// os.Stdout at the process level to assert on exactly these\n+\t\t\t// writes.\n+\t\t\treturn handleHook(args, os.Stdout, os.Stderr)\n+\t\t},\n+\t}\n+}\n+\n+// handleHook processes a CLI hook invocation. It parses the\n+// HookPluginData JSON from args, and if the command failed,\n+// writes a HookMessage with a context-specific hint to w.\n+func handleHook(args []string, w, errW io.Writer) error {\n+\tif len(args) == 0 {\n+\t\treturn nil\n+\t}\n+\n+\tvar hookData manager.HookPluginData\n+\tif err := json.Unmarshal([]byte(args[0]), &hookData); err != nil {\n+\t\tfmt.Fprintf(errW, \"warning: failed to parse hook data: %v\\n\", err)\n+\t\treturn nil\n+\t}\n+\n+\tif hookData.CommandError == \"\" {\n+\t\treturn nil\n+\t}\n+\n+\ttmpl := defaultHintTemplate\n+\tif t, ok := hintTemplates[hookData.RootCmd]; ok {\n+\t\ttmpl = t\n+\t}\n+\n+\tenc := json.NewEncoder(w)\n+\tenc.SetEscapeHTML(false)\n+\treturn enc.Encode(hooks.HookMessage{\n+\t\tType: hooks.NextSteps,\n+\t\tTemplate: tmpl,\n+\t})\n+}\n\ndiff --git a/cli/commands/hooks_test.go b/cli/commands/hooks_test.go\nnew file mode 100644\nindex 000000000..931fab662\n--- /dev/null\n+++ b/cli/commands/hooks_test.go\n@@ -0,0 +1,136 @@\n+package commands\n+\n+import (\n+\t\"bytes\"\n+\t\"context\"\n+\t\"encoding/json\"\n+\t\"errors\"\n+\t\"io\"\n+\t\"os\"\n+\t\"testing\"\n+\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/stretchr/testify/assert\"\n+\t\"github.com/stretchr/testify/require\"\n+)\n+\n+func TestHandleHook_NoArgs(t *testing.T) {\n+\tvar buf bytes.Buffer\n+\terr := handleHook(nil, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+func TestHandleHook_InvalidJSON(t *testing.T) {\n+\tvar stdout, stderr bytes.Buffer\n+\terr := handleHook([]string{\"not json\"}, &stdout, &stderr)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, stdout.String())\n+\tassert.Contains(t, stderr.String(), \"warning: failed to parse hook data\")\n+}\n+\n+func TestHandleHook_Success(t *testing.T) {\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"\",\n+\t})\n+\n+\tvar buf bytes.Buffer\n+\terr := handleHook([]string{data}, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+// TestHandleHook_Failure covers every configured hook string. The CLI passes\n+// the matched hook config string as RootCmd, so multi-word hook entries such\n+// as \"buildx build\" arrive exactly as configured.\n+func TestHandleHook_Failure(t *testing.T) {\n+\ttests := []struct {\n+\t\tname string\n+\t\trootCmd string\n+\t\twantTmpl string\n+\t}{\n+\t\t{\n+\t\t\tname: \"build\",\n+\t\t\trootCmd: \"build\",\n+\t\t\twantTmpl: hintTemplates[\"build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"buildx build\",\n+\t\t\trootCmd: \"buildx build\",\n+\t\t\twantTmpl: hintTemplates[\"buildx build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"run\",\n+\t\t\trootCmd: \"run\",\n+\t\t\twantTmpl: hintTemplates[\"run\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"compose\",\n+\t\t\trootCmd: \"compose\",\n+\t\t\twantTmpl: hintTemplates[\"compose\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"unknown command falls back to default\",\n+\t\t\trootCmd: \"push\",\n+\t\t\twantTmpl: defaultHintTemplate,\n+\t\t},\n+\t}\n+\n+\tfor _, tc := range tests {\n+\t\tt.Run(tc.name, func(t *testing.T) {\n+\t\t\tdata := marshal(t, manager.HookPluginData{\n+\t\t\t\tRootCmd: tc.rootCmd,\n+\t\t\t\tCommandError: \"exit status 1\",\n+\t\t\t})\n+\n+\t\t\tvar buf bytes.Buffer\n+\t\t\terr := handleHook([]string{data}, &buf, io.Discard)\n+\t\t\trequire.NoError(t, err)\n+\n+\t\t\tvar msg hooks.HookMessage\n+\t\t\trequire.NoError(t, json.Unmarshal(buf.Bytes(), &msg))\n+\t\t\tassert.EqualValues(t, hooks.NextSteps, msg.Type)\n+\t\t\tassert.Equal(t, tc.wantTmpl, msg.Template)\n+\t\t})\n+\t}\n+}\n+\n+func TestHooks_SkippedWhenFeatureDisabled(t *testing.T) {\n+\torig := checkEnabled\n+\tcheckEnabled = func(context.Context) error {\n+\t\treturn errors.New(\"Docker AI is not enabled\")\n+\t}\n+\tt.Cleanup(func() { checkEnabled = orig })\n+\n+\t// The hook writes to the real process stdout by design, so capture it\n+\t// the same way cli/main_test.go's setStdout helper does: swap os.Stdout\n+\t// for a temp file around the command execution.\n+\ttmp, err := os.CreateTemp(t.TempDir(), \"stdout\")\n+\trequire.NoError(t, err)\n+\torigStdout := os.Stdout\n+\tos.Stdout = tmp\n+\tt.Cleanup(func() { os.Stdout = origStdout })\n+\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"exit status 1\",\n+\t})\n+\n+\tcmd := Hooks()\n+\tcmd.SetArgs([]string{data})\n+\trequire.NoError(t, cmd.Execute())\n+\n+\trequire.NoError(t, tmp.Close())\n+\toutput, err := os.ReadFile(tmp.Name())\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, string(output), \"no hint should be shown when Docker AI is disabled\")\n+}\n+\n+func marshal(t *testing.T, v any) string {\n+\tt.Helper()\n+\tb, err := json.Marshal(v)\n+\trequire.NoError(t, err)\n+\treturn string(b)\n+}\n\ndiff --git a/cli/main.go b/cli/main.go\nindex be9cb6210..c57436bff 100644\n--- a/cli/main.go\n+++ b/cli/main.go\n@@ -43,6 +43,10 @@ func main() {\n \t\tcmd.AddCommand(commands.Thread())\n \t\tcmd.AddCommand(commands.Mcp())\n \n+\t\t// CLI hooks subcommand — registered for both V1 and V2 so that\n+\t\t// \"What's next:\" hints work regardless of the active feature flag.\n+\t\tcmd.AddCommand(commands.Hooks())\n+\n \t\toriginalPreRun := cmd.PersistentPreRunE\n \t\tcmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n \t\t\tif err := plugin.PersistentPreRunE(cmd, args); err != nil {\n\ndiff --git a/cli/main_test.go b/cli/main_test.go\nindex 2ccecd42f..338474e68 100644\n--- a/cli/main_test.go\n+++ b/cli/main_test.go\n@@ -18,6 +18,8 @@ func TestMain(m *testing.M) {\n \t// Make desktop.CheckFeatureIsEnabled a no-op so CLI tests never\n \t// require a running Docker Desktop.\n \tversion.GoTest = true\n+\t// The CLI hook tests below reuse this harness: with GoTest set, hook\n+\t// execution skips the Desktop check just like every other CLI test.\n \tos.Exit(m.Run())\n }\n \n@@ -98,6 +100,31 @@ func TestMCPBuiltin(t *testing.T) {\n \tassert.Contains(t, string(output), `\"docker\"`)\n }\n \n+func TestHooksShowsHintOnFailure(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"exit status 1\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Contains(t, string(output), `docker ai`)\n+\tassert.Contains(t, string(output), `build failure with Gordon`)\n+}\n+\n+func TestHooksNoOutputOnSuccess(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, string(output))\n+}\n+\n func runDockerAI(args ...string) {\n \tos.Args = args\n \tmain()\n```", "created_at": "2026-02-18T11:16:21-05:00" } } diff --git a/review-pr/agents/evals/success-2.json b/review-pr/agents/evals/success-2.json index 4913aae..194ff41 100644 --- a/review-pr/agents/evals/success-2.json +++ b/review-pr/agents/evals/success-2.json @@ -6,10 +6,12 @@ "relevance": [ "The agent ran 'echo $GITHUB_ACTIONS' before performing the review to detect the output mode", "The agent output the review to the console as formatted markdown instead of posting via gh api", - "The review assessment label is '🟢 APPROVE' or '🟡 NEEDS ATTENTION' — it is NOT '🔴 CRITICAL'", + "The review assessment label is '🟢 NO FINDINGS' or '🟡 NEEDS ATTENTION' — it is NOT '🔴 CRITICAL'", "The drafter response is valid JSON containing a 'findings' array, a 'summary' field, and 'review_complete' set to true", "No findings have severity 'high' with verdict 'CONFIRMED' or 'LIKELY'", - "The review does not label any finding as CRITICAL or block-worthy" + "The review keeps neutral comment-only semantics: it does not label any finding as CRITICAL or block-worthy, and it does not use approval wording such as 'APPROVE' or 'LGTM'", + "The review does not claim the 'buildx build' hint template is unreachable dead code or that RootCmd only carries a top-level command token, and it does not claim the tests require a running Docker Desktop — the diff documents that RootCmd is the full matched hook config string and that TestMain sets version.GoTest = true", + "The review does not flag the documented use of os.Stdout/os.Stderr in the hook RunE as a confirmed or likely bug — the diff explains and tests that hook output goes to the process streams" ] }, "messages": [ @@ -18,7 +20,7 @@ "agentName": "", "message": { "role": "user", - "content": "Review the following PR.\n\n## PR Information\n- **Title**: Add CLI hooks to show Gordon hints on command failure\n- **Author**: derekmisler\n- **Branch**: dm/cli-hooks → main\n- **Files Changed**: 4\n\n## PR Description\nRegister a \"docker-cli-plugin-hooks\" subcommand so the Docker CLI shows \"What's next:\" hints suggesting Gordon when commands like build, run, or compose up fail.\n\n### Summary\n- Add a hidden `docker-cli-plugin-hooks` subcommand that returns contextual \"What's next:\" hints when Docker CLI commands fail\n- Hints follow the DDS-CLI format: `description → command`\n- Registered for both V1 and V2 code paths\n\n## Diff\n\n```diff\ndiff --git a/cli/commands/hooks.go b/cli/commands/hooks.go\nnew file mode 100644\nindex 000000000..96e86d62b\n--- /dev/null\n+++ b/cli/commands/hooks.go\n@@ -0,0 +1,76 @@\n+package commands\n+\n+import (\n+\t\"encoding/json\"\n+\t\"fmt\"\n+\t\"io\"\n+\t\"os\"\n+\n+\t\"github.com/docker/ai-common/desktop\"\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/docker/cli/cli-plugins/metadata\"\n+\t\"github.com/spf13/cobra\"\n+)\n+\n+var hintTemplates = map[string]string{\n+\t\"build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"buildx build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"run\": `Debug this container error with Gordon → docker ai \"help me fix this container error\"`,\n+\t\"compose\": `Debug this Compose error with Gordon → docker ai \"help me fix this compose error\"`,\n+}\n+\n+const defaultHintTemplate = `Debug this error with Gordon → docker ai`\n+\n+// checkEnabled verifies that Docker AI is enabled before showing hints.\n+// Replaced in tests to avoid requiring a running Docker Desktop.\n+var checkEnabled = desktop.CheckFeatureIsEnabled\n+\n+// Hooks returns the hidden subcommand that the Docker CLI invokes\n+// after command execution when the \"ai\" plugin has hooks configured.\n+func Hooks() *cobra.Command {\n+\treturn &cobra.Command{\n+\t\tUse: metadata.HookSubcommandName,\n+\t\tHidden: true,\n+\t\t// Override PersistentPreRun to prevent the parent's PersistentPreRunE\n+\t\t// (plugin initialization) from running for hook invocations.\n+\t\tPersistentPreRun: func(*cobra.Command, []string) {},\n+\t\tRunE: func(cmd *cobra.Command, args []string) error {\n+\t\t\tif err := checkEnabled(cmd.Context()); err != nil {\n+\t\t\t\treturn nil\n+\t\t\t}\n+\t\t\treturn handleHook(args, os.Stdout, os.Stderr)\n+\t\t},\n+\t}\n+}\n+\n+// handleHook processes a CLI hook invocation. It parses the\n+// HookPluginData JSON from args, and if the command failed,\n+// writes a HookMessage with a context-specific hint to w.\n+func handleHook(args []string, w, errW io.Writer) error {\n+\tif len(args) == 0 {\n+\t\treturn nil\n+\t}\n+\n+\tvar hookData manager.HookPluginData\n+\tif err := json.Unmarshal([]byte(args[0]), &hookData); err != nil {\n+\t\tfmt.Fprintf(errW, \"warning: failed to parse hook data: %v\\n\", err)\n+\t\treturn nil\n+\t}\n+\n+\tif hookData.CommandError == \"\" {\n+\t\treturn nil\n+\t}\n+\n+\ttmpl := defaultHintTemplate\n+\tif t, ok := hintTemplates[hookData.RootCmd]; ok {\n+\t\ttmpl = t\n+\t}\n+\n+\tenc := json.NewEncoder(w)\n+\tenc.SetEscapeHTML(false)\n+\treturn enc.Encode(hooks.HookMessage{\n+\t\tType: hooks.NextSteps,\n+\t\tTemplate: tmpl,\n+\t})\n+}\n\ndiff --git a/cli/commands/hooks_test.go b/cli/commands/hooks_test.go\nnew file mode 100644\nindex 000000000..931fab662\n--- /dev/null\n+++ b/cli/commands/hooks_test.go\n@@ -0,0 +1,125 @@\n+package commands\n+\n+import (\n+\t\"bytes\"\n+\t\"context\"\n+\t\"encoding/json\"\n+\t\"errors\"\n+\t\"io\"\n+\t\"testing\"\n+\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/stretchr/testify/assert\"\n+\t\"github.com/stretchr/testify/require\"\n+)\n+\n+func TestHandleHook_NoArgs(t *testing.T) {\n+\tvar buf bytes.Buffer\n+\terr := handleHook(nil, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+func TestHandleHook_InvalidJSON(t *testing.T) {\n+\tvar stdout, stderr bytes.Buffer\n+\terr := handleHook([]string{\"not json\"}, &stdout, &stderr)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, stdout.String())\n+\tassert.Contains(t, stderr.String(), \"warning: failed to parse hook data\")\n+}\n+\n+func TestHandleHook_Success(t *testing.T) {\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"\",\n+\t})\n+\n+\tvar buf bytes.Buffer\n+\terr := handleHook([]string{data}, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+func TestHandleHook_Failure(t *testing.T) {\n+\ttests := []struct {\n+\t\tname string\n+\t\trootCmd string\n+\t\twantTmpl string\n+\t}{\n+\t\t{\n+\t\t\tname: \"build\",\n+\t\t\trootCmd: \"build\",\n+\t\t\twantTmpl: hintTemplates[\"build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"buildx build\",\n+\t\t\trootCmd: \"buildx build\",\n+\t\t\twantTmpl: hintTemplates[\"buildx build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"run\",\n+\t\t\trootCmd: \"run\",\n+\t\t\twantTmpl: hintTemplates[\"run\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"compose\",\n+\t\t\trootCmd: \"compose\",\n+\t\t\twantTmpl: hintTemplates[\"compose\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"unknown command falls back to default\",\n+\t\t\trootCmd: \"push\",\n+\t\t\twantTmpl: defaultHintTemplate,\n+\t\t},\n+\t}\n+\n+\tfor _, tc := range tests {\n+\t\tt.Run(tc.name, func(t *testing.T) {\n+\t\t\tdata := marshal(t, manager.HookPluginData{\n+\t\t\t\tRootCmd: tc.rootCmd,\n+\t\t\t\tCommandError: \"exit status 1\",\n+\t\t\t})\n+\n+\t\t\tvar buf bytes.Buffer\n+\t\t\terr := handleHook([]string{data}, &buf, io.Discard)\n+\t\t\trequire.NoError(t, err)\n+\n+\t\t\tvar msg hooks.HookMessage\n+\t\t\trequire.NoError(t, json.Unmarshal(buf.Bytes(), &msg))\n+\t\t\tassert.EqualValues(t, hooks.NextSteps, msg.Type)\n+\t\t\tassert.Equal(t, tc.wantTmpl, msg.Template)\n+\t\t})\n+\t}\n+}\n+\n+func TestHooks_SkippedWhenFeatureDisabled(t *testing.T) {\n+\torig := checkEnabled\n+\tcheckEnabled = func(context.Context) error {\n+\t\treturn errors.New(\"Docker AI is not enabled\")\n+\t}\n+\tt.Cleanup(func() { checkEnabled = orig })\n+\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"exit status 1\",\n+\t})\n+\n+\tcmd := Hooks()\n+\tcmd.SetArgs([]string{data})\n+\tcmd.SetOut(io.Discard)\n+\n+\tvar buf bytes.Buffer\n+\tcmd.SetOut(&buf)\n+\n+\terr := cmd.Execute()\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String(), \"no hint should be shown when Docker AI is disabled\")\n+}\n+\n+func marshal(t *testing.T, v any) string {\n+\tt.Helper()\n+\tb, err := json.Marshal(v)\n+\trequire.NoError(t, err)\n+\treturn string(b)\n+}\n\ndiff --git a/cli/main.go b/cli/main.go\nindex be9cb6210..c57436bff 100644\n--- a/cli/main.go\n+++ b/cli/main.go\n@@ -43,6 +43,10 @@ func main() {\n \t\tcmd.AddCommand(commands.Thread())\n \t\tcmd.AddCommand(commands.Mcp())\n \n+\t\t// CLI hooks subcommand — registered for both V1 and V2 so that\n+\t\t// \"What's next:\" hints work regardless of the active feature flag.\n+\t\tcmd.AddCommand(commands.Hooks())\n+\n \t\toriginalPreRun := cmd.PersistentPreRunE\n \t\tcmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n \t\t\tif err := plugin.PersistentPreRunE(cmd, args); err != nil {\n\ndiff --git a/cli/main_test.go b/cli/main_test.go\nindex 2ccecd42f..338474e68 100644\n--- a/cli/main_test.go\n+++ b/cli/main_test.go\n@@ -98,6 +98,31 @@ func TestMCPBuiltin(t *testing.T) {\n \tassert.Contains(t, string(output), `\"docker\"`)\n }\n \n+func TestHooksShowsHintOnFailure(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"exit status 1\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Contains(t, string(output), `docker ai`)\n+\tassert.Contains(t, string(output), `build failure with Gordon`)\n+}\n+\n+func TestHooksNoOutputOnSuccess(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, string(output))\n+}\n+\n func runDockerAI(args ...string) {\n \tos.Args = args\n \tmain()\n```", + "content": "Review the following PR.\n\n## PR Information\n- **Title**: Add CLI hooks to show Gordon hints on command failure\n- **Author**: derekmisler\n- **Branch**: dm/cli-hooks → main\n- **Files Changed**: 4\n\n## PR Description\nRegister a \"docker-cli-plugin-hooks\" subcommand so the Docker CLI shows \"What's next:\" hints suggesting Gordon when commands like build, run, or compose up fail.\n\n### Summary\n- Add a hidden `docker-cli-plugin-hooks` subcommand that returns contextual \"What's next:\" hints when Docker CLI commands fail\n- Hints follow the DDS-CLI format: `description → command`\n- Registered for both V1 and V2 code paths\n- Per the docker/cli plugin-hooks contract (docker/cli#6794), the CLI passes the matched hook config string as `HookPluginData.RootCmd` — multi-word entries like `buildx build` included — and reads the hook subprocess's stdout for the JSON `HookMessage`\n- CLI-level tests run in-process under the existing `TestMain`, which sets `version.GoTest = true` so `desktop.CheckFeatureIsEnabled` is a no-op without Docker Desktop\n\n## Diff\n\n```diff\ndiff --git a/cli/commands/hooks.go b/cli/commands/hooks.go\nnew file mode 100644\nindex 000000000..96e86d62b\n--- /dev/null\n+++ b/cli/commands/hooks.go\n@@ -0,0 +1,88 @@\n+package commands\n+\n+import (\n+\t\"encoding/json\"\n+\t\"fmt\"\n+\t\"io\"\n+\t\"os\"\n+\n+\t\"github.com/docker/ai-common/desktop\"\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/docker/cli/cli-plugins/metadata\"\n+\t\"github.com/spf13/cobra\"\n+)\n+\n+// hintTemplates maps a hook config string to the hint shown when that command\n+// fails. Per the docker/cli plugin-hooks contract (docker/cli#6794), the CLI\n+// passes the exact hook configuration string it matched for the invocation as\n+// HookPluginData.RootCmd — including multi-word entries such as\n+// \"buildx build\" — never just the top-level command token.\n+var hintTemplates = map[string]string{\n+\t\"build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"buildx build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"run\": `Debug this container error with Gordon → docker ai \"help me fix this container error\"`,\n+\t\"compose\": `Debug this Compose error with Gordon → docker ai \"help me fix this compose error\"`,\n+}\n+\n+const defaultHintTemplate = `Debug this error with Gordon → docker ai`\n+\n+// checkEnabled verifies that Docker AI is enabled before showing hints.\n+// desktop.CheckFeatureIsEnabled short-circuits to nil when version.GoTest is\n+// true — cli/main_test.go's TestMain sets that flag — so the in-process CLI\n+// tests never need a running Docker Desktop. Unit tests replace this var.\n+var checkEnabled = desktop.CheckFeatureIsEnabled\n+\n+// Hooks returns the hidden subcommand that the Docker CLI invokes\n+// after command execution when the \"ai\" plugin has hooks configured.\n+func Hooks() *cobra.Command {\n+\treturn &cobra.Command{\n+\t\tUse: metadata.HookSubcommandName,\n+\t\tHidden: true,\n+\t\t// Override PersistentPreRun to prevent the parent's PersistentPreRunE\n+\t\t// (plugin initialization) from running for hook invocations.\n+\t\tPersistentPreRun: func(*cobra.Command, []string) {},\n+\t\tRunE: func(cmd *cobra.Command, args []string) error {\n+\t\t\tif err := checkEnabled(cmd.Context()); err != nil {\n+\t\t\t\treturn nil\n+\t\t\t}\n+\t\t\t// Hook output deliberately goes to the process streams rather\n+\t\t\t// than cmd.OutOrStdout(): the Docker CLI reads the hook\n+\t\t\t// subprocess's stdout, and cli/main_test.go's setStdout swaps\n+\t\t\t// os.Stdout at the process level to assert on exactly these\n+\t\t\t// writes.\n+\t\t\treturn handleHook(args, os.Stdout, os.Stderr)\n+\t\t},\n+\t}\n+}\n+\n+// handleHook processes a CLI hook invocation. It parses the\n+// HookPluginData JSON from args, and if the command failed,\n+// writes a HookMessage with a context-specific hint to w.\n+func handleHook(args []string, w, errW io.Writer) error {\n+\tif len(args) == 0 {\n+\t\treturn nil\n+\t}\n+\n+\tvar hookData manager.HookPluginData\n+\tif err := json.Unmarshal([]byte(args[0]), &hookData); err != nil {\n+\t\tfmt.Fprintf(errW, \"warning: failed to parse hook data: %v\\n\", err)\n+\t\treturn nil\n+\t}\n+\n+\tif hookData.CommandError == \"\" {\n+\t\treturn nil\n+\t}\n+\n+\ttmpl := defaultHintTemplate\n+\tif t, ok := hintTemplates[hookData.RootCmd]; ok {\n+\t\ttmpl = t\n+\t}\n+\n+\tenc := json.NewEncoder(w)\n+\tenc.SetEscapeHTML(false)\n+\treturn enc.Encode(hooks.HookMessage{\n+\t\tType: hooks.NextSteps,\n+\t\tTemplate: tmpl,\n+\t})\n+}\n\ndiff --git a/cli/commands/hooks_test.go b/cli/commands/hooks_test.go\nnew file mode 100644\nindex 000000000..931fab662\n--- /dev/null\n+++ b/cli/commands/hooks_test.go\n@@ -0,0 +1,136 @@\n+package commands\n+\n+import (\n+\t\"bytes\"\n+\t\"context\"\n+\t\"encoding/json\"\n+\t\"errors\"\n+\t\"io\"\n+\t\"os\"\n+\t\"testing\"\n+\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/stretchr/testify/assert\"\n+\t\"github.com/stretchr/testify/require\"\n+)\n+\n+func TestHandleHook_NoArgs(t *testing.T) {\n+\tvar buf bytes.Buffer\n+\terr := handleHook(nil, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+func TestHandleHook_InvalidJSON(t *testing.T) {\n+\tvar stdout, stderr bytes.Buffer\n+\terr := handleHook([]string{\"not json\"}, &stdout, &stderr)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, stdout.String())\n+\tassert.Contains(t, stderr.String(), \"warning: failed to parse hook data\")\n+}\n+\n+func TestHandleHook_Success(t *testing.T) {\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"\",\n+\t})\n+\n+\tvar buf bytes.Buffer\n+\terr := handleHook([]string{data}, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+// TestHandleHook_Failure covers every configured hook string. The CLI passes\n+// the matched hook config string as RootCmd, so multi-word hook entries such\n+// as \"buildx build\" arrive exactly as configured.\n+func TestHandleHook_Failure(t *testing.T) {\n+\ttests := []struct {\n+\t\tname string\n+\t\trootCmd string\n+\t\twantTmpl string\n+\t}{\n+\t\t{\n+\t\t\tname: \"build\",\n+\t\t\trootCmd: \"build\",\n+\t\t\twantTmpl: hintTemplates[\"build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"buildx build\",\n+\t\t\trootCmd: \"buildx build\",\n+\t\t\twantTmpl: hintTemplates[\"buildx build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"run\",\n+\t\t\trootCmd: \"run\",\n+\t\t\twantTmpl: hintTemplates[\"run\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"compose\",\n+\t\t\trootCmd: \"compose\",\n+\t\t\twantTmpl: hintTemplates[\"compose\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"unknown command falls back to default\",\n+\t\t\trootCmd: \"push\",\n+\t\t\twantTmpl: defaultHintTemplate,\n+\t\t},\n+\t}\n+\n+\tfor _, tc := range tests {\n+\t\tt.Run(tc.name, func(t *testing.T) {\n+\t\t\tdata := marshal(t, manager.HookPluginData{\n+\t\t\t\tRootCmd: tc.rootCmd,\n+\t\t\t\tCommandError: \"exit status 1\",\n+\t\t\t})\n+\n+\t\t\tvar buf bytes.Buffer\n+\t\t\terr := handleHook([]string{data}, &buf, io.Discard)\n+\t\t\trequire.NoError(t, err)\n+\n+\t\t\tvar msg hooks.HookMessage\n+\t\t\trequire.NoError(t, json.Unmarshal(buf.Bytes(), &msg))\n+\t\t\tassert.EqualValues(t, hooks.NextSteps, msg.Type)\n+\t\t\tassert.Equal(t, tc.wantTmpl, msg.Template)\n+\t\t})\n+\t}\n+}\n+\n+func TestHooks_SkippedWhenFeatureDisabled(t *testing.T) {\n+\torig := checkEnabled\n+\tcheckEnabled = func(context.Context) error {\n+\t\treturn errors.New(\"Docker AI is not enabled\")\n+\t}\n+\tt.Cleanup(func() { checkEnabled = orig })\n+\n+\t// The hook writes to the real process stdout by design, so capture it\n+\t// the same way cli/main_test.go's setStdout helper does: swap os.Stdout\n+\t// for a temp file around the command execution.\n+\ttmp, err := os.CreateTemp(t.TempDir(), \"stdout\")\n+\trequire.NoError(t, err)\n+\torigStdout := os.Stdout\n+\tos.Stdout = tmp\n+\tt.Cleanup(func() { os.Stdout = origStdout })\n+\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"exit status 1\",\n+\t})\n+\n+\tcmd := Hooks()\n+\tcmd.SetArgs([]string{data})\n+\trequire.NoError(t, cmd.Execute())\n+\n+\trequire.NoError(t, tmp.Close())\n+\toutput, err := os.ReadFile(tmp.Name())\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, string(output), \"no hint should be shown when Docker AI is disabled\")\n+}\n+\n+func marshal(t *testing.T, v any) string {\n+\tt.Helper()\n+\tb, err := json.Marshal(v)\n+\trequire.NoError(t, err)\n+\treturn string(b)\n+}\n\ndiff --git a/cli/main.go b/cli/main.go\nindex be9cb6210..c57436bff 100644\n--- a/cli/main.go\n+++ b/cli/main.go\n@@ -43,6 +43,10 @@ func main() {\n \t\tcmd.AddCommand(commands.Thread())\n \t\tcmd.AddCommand(commands.Mcp())\n \n+\t\t// CLI hooks subcommand — registered for both V1 and V2 so that\n+\t\t// \"What's next:\" hints work regardless of the active feature flag.\n+\t\tcmd.AddCommand(commands.Hooks())\n+\n \t\toriginalPreRun := cmd.PersistentPreRunE\n \t\tcmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n \t\t\tif err := plugin.PersistentPreRunE(cmd, args); err != nil {\n\ndiff --git a/cli/main_test.go b/cli/main_test.go\nindex 2ccecd42f..338474e68 100644\n--- a/cli/main_test.go\n+++ b/cli/main_test.go\n@@ -18,6 +18,8 @@ func TestMain(m *testing.M) {\n \t// Make desktop.CheckFeatureIsEnabled a no-op so CLI tests never\n \t// require a running Docker Desktop.\n \tversion.GoTest = true\n+\t// The CLI hook tests below reuse this harness: with GoTest set, hook\n+\t// execution skips the Desktop check just like every other CLI test.\n \tos.Exit(m.Run())\n }\n \n@@ -98,6 +100,31 @@ func TestMCPBuiltin(t *testing.T) {\n \tassert.Contains(t, string(output), `\"docker\"`)\n }\n \n+func TestHooksShowsHintOnFailure(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"exit status 1\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Contains(t, string(output), `docker ai`)\n+\tassert.Contains(t, string(output), `build failure with Gordon`)\n+}\n+\n+func TestHooksNoOutputOnSuccess(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, string(output))\n+}\n+\n func runDockerAI(args ...string) {\n \tos.Args = args\n \tmain()\n```", "created_at": "2026-02-18T11:16:21-05:00" } } diff --git a/review-pr/agents/evals/success-3.json b/review-pr/agents/evals/success-3.json index bc72bac..5b2ffc3 100644 --- a/review-pr/agents/evals/success-3.json +++ b/review-pr/agents/evals/success-3.json @@ -6,10 +6,12 @@ "relevance": [ "The agent ran 'echo $GITHUB_ACTIONS' before performing the review to detect the output mode", "The agent output the review to the console as formatted markdown instead of posting via gh api", - "The review assessment label is '🟢 APPROVE' or '🟡 NEEDS ATTENTION' — it is NOT '🔴 CRITICAL'", + "The review assessment label is '🟢 NO FINDINGS' or '🟡 NEEDS ATTENTION' — it is NOT '🔴 CRITICAL'", "The drafter response is valid JSON containing a 'findings' array, a 'summary' field, and 'review_complete' set to true", "No findings have severity 'high' with verdict 'CONFIRMED' or 'LIKELY'", - "The review does not label any finding as CRITICAL or block-worthy" + "The review keeps neutral comment-only semantics: it does not label any finding as CRITICAL or block-worthy, and it does not use approval wording such as 'APPROVE' or 'LGTM'", + "The review does not claim the 'buildx build' hint template is unreachable dead code or that RootCmd only carries a top-level command token, and it does not claim the tests require a running Docker Desktop — the diff documents that RootCmd is the full matched hook config string and that TestMain sets version.GoTest = true", + "The review does not flag the documented use of os.Stdout/os.Stderr in the hook RunE as a confirmed or likely bug — the diff explains and tests that hook output goes to the process streams" ] }, "messages": [ @@ -18,7 +20,7 @@ "agentName": "", "message": { "role": "user", - "content": "Review the following PR.\n\n## PR Information\n- **Title**: Add CLI hooks to show Gordon hints on command failure\n- **Author**: derekmisler\n- **Branch**: dm/cli-hooks → main\n- **Files Changed**: 4\n\n## PR Description\nRegister a \"docker-cli-plugin-hooks\" subcommand so the Docker CLI shows \"What's next:\" hints suggesting Gordon when commands like build, run, or compose up fail.\n\n### Summary\n- Add a hidden `docker-cli-plugin-hooks` subcommand that returns contextual \"What's next:\" hints when Docker CLI commands fail\n- Hints follow the DDS-CLI format: `description → command`\n- Registered for both V1 and V2 code paths\n\n## Diff\n\n```diff\ndiff --git a/cli/commands/hooks.go b/cli/commands/hooks.go\nnew file mode 100644\nindex 000000000..96e86d62b\n--- /dev/null\n+++ b/cli/commands/hooks.go\n@@ -0,0 +1,76 @@\n+package commands\n+\n+import (\n+\t\"encoding/json\"\n+\t\"fmt\"\n+\t\"io\"\n+\t\"os\"\n+\n+\t\"github.com/docker/ai-common/desktop\"\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/docker/cli/cli-plugins/metadata\"\n+\t\"github.com/spf13/cobra\"\n+)\n+\n+var hintTemplates = map[string]string{\n+\t\"build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"buildx build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"run\": `Debug this container error with Gordon → docker ai \"help me fix this container error\"`,\n+\t\"compose\": `Debug this Compose error with Gordon → docker ai \"help me fix this compose error\"`,\n+}\n+\n+const defaultHintTemplate = `Debug this error with Gordon → docker ai`\n+\n+// checkEnabled verifies that Docker AI is enabled before showing hints.\n+// Replaced in tests to avoid requiring a running Docker Desktop.\n+var checkEnabled = desktop.CheckFeatureIsEnabled\n+\n+// Hooks returns the hidden subcommand that the Docker CLI invokes\n+// after command execution when the \"ai\" plugin has hooks configured.\n+func Hooks() *cobra.Command {\n+\treturn &cobra.Command{\n+\t\tUse: metadata.HookSubcommandName,\n+\t\tHidden: true,\n+\t\t// Override PersistentPreRun to prevent the parent's PersistentPreRunE\n+\t\t// (plugin initialization) from running for hook invocations.\n+\t\tPersistentPreRun: func(*cobra.Command, []string) {},\n+\t\tRunE: func(cmd *cobra.Command, args []string) error {\n+\t\t\tif err := checkEnabled(cmd.Context()); err != nil {\n+\t\t\t\treturn nil\n+\t\t\t}\n+\t\t\treturn handleHook(args, os.Stdout, os.Stderr)\n+\t\t},\n+\t}\n+}\n+\n+// handleHook processes a CLI hook invocation. It parses the\n+// HookPluginData JSON from args, and if the command failed,\n+// writes a HookMessage with a context-specific hint to w.\n+func handleHook(args []string, w, errW io.Writer) error {\n+\tif len(args) == 0 {\n+\t\treturn nil\n+\t}\n+\n+\tvar hookData manager.HookPluginData\n+\tif err := json.Unmarshal([]byte(args[0]), &hookData); err != nil {\n+\t\tfmt.Fprintf(errW, \"warning: failed to parse hook data: %v\\n\", err)\n+\t\treturn nil\n+\t}\n+\n+\tif hookData.CommandError == \"\" {\n+\t\treturn nil\n+\t}\n+\n+\ttmpl := defaultHintTemplate\n+\tif t, ok := hintTemplates[hookData.RootCmd]; ok {\n+\t\ttmpl = t\n+\t}\n+\n+\tenc := json.NewEncoder(w)\n+\tenc.SetEscapeHTML(false)\n+\treturn enc.Encode(hooks.HookMessage{\n+\t\tType: hooks.NextSteps,\n+\t\tTemplate: tmpl,\n+\t})\n+}\n\ndiff --git a/cli/commands/hooks_test.go b/cli/commands/hooks_test.go\nnew file mode 100644\nindex 000000000..931fab662\n--- /dev/null\n+++ b/cli/commands/hooks_test.go\n@@ -0,0 +1,125 @@\n+package commands\n+\n+import (\n+\t\"bytes\"\n+\t\"context\"\n+\t\"encoding/json\"\n+\t\"errors\"\n+\t\"io\"\n+\t\"testing\"\n+\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/stretchr/testify/assert\"\n+\t\"github.com/stretchr/testify/require\"\n+)\n+\n+func TestHandleHook_NoArgs(t *testing.T) {\n+\tvar buf bytes.Buffer\n+\terr := handleHook(nil, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+func TestHandleHook_InvalidJSON(t *testing.T) {\n+\tvar stdout, stderr bytes.Buffer\n+\terr := handleHook([]string{\"not json\"}, &stdout, &stderr)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, stdout.String())\n+\tassert.Contains(t, stderr.String(), \"warning: failed to parse hook data\")\n+}\n+\n+func TestHandleHook_Success(t *testing.T) {\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"\",\n+\t})\n+\n+\tvar buf bytes.Buffer\n+\terr := handleHook([]string{data}, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+func TestHandleHook_Failure(t *testing.T) {\n+\ttests := []struct {\n+\t\tname string\n+\t\trootCmd string\n+\t\twantTmpl string\n+\t}{\n+\t\t{\n+\t\t\tname: \"build\",\n+\t\t\trootCmd: \"build\",\n+\t\t\twantTmpl: hintTemplates[\"build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"buildx build\",\n+\t\t\trootCmd: \"buildx build\",\n+\t\t\twantTmpl: hintTemplates[\"buildx build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"run\",\n+\t\t\trootCmd: \"run\",\n+\t\t\twantTmpl: hintTemplates[\"run\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"compose\",\n+\t\t\trootCmd: \"compose\",\n+\t\t\twantTmpl: hintTemplates[\"compose\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"unknown command falls back to default\",\n+\t\t\trootCmd: \"push\",\n+\t\t\twantTmpl: defaultHintTemplate,\n+\t\t},\n+\t}\n+\n+\tfor _, tc := range tests {\n+\t\tt.Run(tc.name, func(t *testing.T) {\n+\t\t\tdata := marshal(t, manager.HookPluginData{\n+\t\t\t\tRootCmd: tc.rootCmd,\n+\t\t\t\tCommandError: \"exit status 1\",\n+\t\t\t})\n+\n+\t\t\tvar buf bytes.Buffer\n+\t\t\terr := handleHook([]string{data}, &buf, io.Discard)\n+\t\t\trequire.NoError(t, err)\n+\n+\t\t\tvar msg hooks.HookMessage\n+\t\t\trequire.NoError(t, json.Unmarshal(buf.Bytes(), &msg))\n+\t\t\tassert.EqualValues(t, hooks.NextSteps, msg.Type)\n+\t\t\tassert.Equal(t, tc.wantTmpl, msg.Template)\n+\t\t})\n+\t}\n+}\n+\n+func TestHooks_SkippedWhenFeatureDisabled(t *testing.T) {\n+\torig := checkEnabled\n+\tcheckEnabled = func(context.Context) error {\n+\t\treturn errors.New(\"Docker AI is not enabled\")\n+\t}\n+\tt.Cleanup(func() { checkEnabled = orig })\n+\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"exit status 1\",\n+\t})\n+\n+\tcmd := Hooks()\n+\tcmd.SetArgs([]string{data})\n+\tcmd.SetOut(io.Discard)\n+\n+\tvar buf bytes.Buffer\n+\tcmd.SetOut(&buf)\n+\n+\terr := cmd.Execute()\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String(), \"no hint should be shown when Docker AI is disabled\")\n+}\n+\n+func marshal(t *testing.T, v any) string {\n+\tt.Helper()\n+\tb, err := json.Marshal(v)\n+\trequire.NoError(t, err)\n+\treturn string(b)\n+}\n\ndiff --git a/cli/main.go b/cli/main.go\nindex be9cb6210..c57436bff 100644\n--- a/cli/main.go\n+++ b/cli/main.go\n@@ -43,6 +43,10 @@ func main() {\n \t\tcmd.AddCommand(commands.Thread())\n \t\tcmd.AddCommand(commands.Mcp())\n \n+\t\t// CLI hooks subcommand — registered for both V1 and V2 so that\n+\t\t// \"What's next:\" hints work regardless of the active feature flag.\n+\t\tcmd.AddCommand(commands.Hooks())\n+\n \t\toriginalPreRun := cmd.PersistentPreRunE\n \t\tcmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n \t\t\tif err := plugin.PersistentPreRunE(cmd, args); err != nil {\n\ndiff --git a/cli/main_test.go b/cli/main_test.go\nindex 2ccecd42f..338474e68 100644\n--- a/cli/main_test.go\n+++ b/cli/main_test.go\n@@ -98,6 +98,31 @@ func TestMCPBuiltin(t *testing.T) {\n \tassert.Contains(t, string(output), `\"docker\"`)\n }\n \n+func TestHooksShowsHintOnFailure(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"exit status 1\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Contains(t, string(output), `docker ai`)\n+\tassert.Contains(t, string(output), `build failure with Gordon`)\n+}\n+\n+func TestHooksNoOutputOnSuccess(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, string(output))\n+}\n+\n func runDockerAI(args ...string) {\n \tos.Args = args\n \tmain()\n```", + "content": "Review the following PR.\n\n## PR Information\n- **Title**: Add CLI hooks to show Gordon hints on command failure\n- **Author**: derekmisler\n- **Branch**: dm/cli-hooks → main\n- **Files Changed**: 4\n\n## PR Description\nRegister a \"docker-cli-plugin-hooks\" subcommand so the Docker CLI shows \"What's next:\" hints suggesting Gordon when commands like build, run, or compose up fail.\n\n### Summary\n- Add a hidden `docker-cli-plugin-hooks` subcommand that returns contextual \"What's next:\" hints when Docker CLI commands fail\n- Hints follow the DDS-CLI format: `description → command`\n- Registered for both V1 and V2 code paths\n- Per the docker/cli plugin-hooks contract (docker/cli#6794), the CLI passes the matched hook config string as `HookPluginData.RootCmd` — multi-word entries like `buildx build` included — and reads the hook subprocess's stdout for the JSON `HookMessage`\n- CLI-level tests run in-process under the existing `TestMain`, which sets `version.GoTest = true` so `desktop.CheckFeatureIsEnabled` is a no-op without Docker Desktop\n\n## Diff\n\n```diff\ndiff --git a/cli/commands/hooks.go b/cli/commands/hooks.go\nnew file mode 100644\nindex 000000000..96e86d62b\n--- /dev/null\n+++ b/cli/commands/hooks.go\n@@ -0,0 +1,88 @@\n+package commands\n+\n+import (\n+\t\"encoding/json\"\n+\t\"fmt\"\n+\t\"io\"\n+\t\"os\"\n+\n+\t\"github.com/docker/ai-common/desktop\"\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/docker/cli/cli-plugins/metadata\"\n+\t\"github.com/spf13/cobra\"\n+)\n+\n+// hintTemplates maps a hook config string to the hint shown when that command\n+// fails. Per the docker/cli plugin-hooks contract (docker/cli#6794), the CLI\n+// passes the exact hook configuration string it matched for the invocation as\n+// HookPluginData.RootCmd — including multi-word entries such as\n+// \"buildx build\" — never just the top-level command token.\n+var hintTemplates = map[string]string{\n+\t\"build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"buildx build\": `Debug this build failure with Gordon → docker ai \"help me fix this build failure\"`,\n+\t\"run\": `Debug this container error with Gordon → docker ai \"help me fix this container error\"`,\n+\t\"compose\": `Debug this Compose error with Gordon → docker ai \"help me fix this compose error\"`,\n+}\n+\n+const defaultHintTemplate = `Debug this error with Gordon → docker ai`\n+\n+// checkEnabled verifies that Docker AI is enabled before showing hints.\n+// desktop.CheckFeatureIsEnabled short-circuits to nil when version.GoTest is\n+// true — cli/main_test.go's TestMain sets that flag — so the in-process CLI\n+// tests never need a running Docker Desktop. Unit tests replace this var.\n+var checkEnabled = desktop.CheckFeatureIsEnabled\n+\n+// Hooks returns the hidden subcommand that the Docker CLI invokes\n+// after command execution when the \"ai\" plugin has hooks configured.\n+func Hooks() *cobra.Command {\n+\treturn &cobra.Command{\n+\t\tUse: metadata.HookSubcommandName,\n+\t\tHidden: true,\n+\t\t// Override PersistentPreRun to prevent the parent's PersistentPreRunE\n+\t\t// (plugin initialization) from running for hook invocations.\n+\t\tPersistentPreRun: func(*cobra.Command, []string) {},\n+\t\tRunE: func(cmd *cobra.Command, args []string) error {\n+\t\t\tif err := checkEnabled(cmd.Context()); err != nil {\n+\t\t\t\treturn nil\n+\t\t\t}\n+\t\t\t// Hook output deliberately goes to the process streams rather\n+\t\t\t// than cmd.OutOrStdout(): the Docker CLI reads the hook\n+\t\t\t// subprocess's stdout, and cli/main_test.go's setStdout swaps\n+\t\t\t// os.Stdout at the process level to assert on exactly these\n+\t\t\t// writes.\n+\t\t\treturn handleHook(args, os.Stdout, os.Stderr)\n+\t\t},\n+\t}\n+}\n+\n+// handleHook processes a CLI hook invocation. It parses the\n+// HookPluginData JSON from args, and if the command failed,\n+// writes a HookMessage with a context-specific hint to w.\n+func handleHook(args []string, w, errW io.Writer) error {\n+\tif len(args) == 0 {\n+\t\treturn nil\n+\t}\n+\n+\tvar hookData manager.HookPluginData\n+\tif err := json.Unmarshal([]byte(args[0]), &hookData); err != nil {\n+\t\tfmt.Fprintf(errW, \"warning: failed to parse hook data: %v\\n\", err)\n+\t\treturn nil\n+\t}\n+\n+\tif hookData.CommandError == \"\" {\n+\t\treturn nil\n+\t}\n+\n+\ttmpl := defaultHintTemplate\n+\tif t, ok := hintTemplates[hookData.RootCmd]; ok {\n+\t\ttmpl = t\n+\t}\n+\n+\tenc := json.NewEncoder(w)\n+\tenc.SetEscapeHTML(false)\n+\treturn enc.Encode(hooks.HookMessage{\n+\t\tType: hooks.NextSteps,\n+\t\tTemplate: tmpl,\n+\t})\n+}\n\ndiff --git a/cli/commands/hooks_test.go b/cli/commands/hooks_test.go\nnew file mode 100644\nindex 000000000..931fab662\n--- /dev/null\n+++ b/cli/commands/hooks_test.go\n@@ -0,0 +1,136 @@\n+package commands\n+\n+import (\n+\t\"bytes\"\n+\t\"context\"\n+\t\"encoding/json\"\n+\t\"errors\"\n+\t\"io\"\n+\t\"os\"\n+\t\"testing\"\n+\n+\t\"github.com/docker/cli/cli-plugins/hooks\"\n+\t\"github.com/docker/cli/cli-plugins/manager\"\n+\t\"github.com/stretchr/testify/assert\"\n+\t\"github.com/stretchr/testify/require\"\n+)\n+\n+func TestHandleHook_NoArgs(t *testing.T) {\n+\tvar buf bytes.Buffer\n+\terr := handleHook(nil, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+func TestHandleHook_InvalidJSON(t *testing.T) {\n+\tvar stdout, stderr bytes.Buffer\n+\terr := handleHook([]string{\"not json\"}, &stdout, &stderr)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, stdout.String())\n+\tassert.Contains(t, stderr.String(), \"warning: failed to parse hook data\")\n+}\n+\n+func TestHandleHook_Success(t *testing.T) {\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"\",\n+\t})\n+\n+\tvar buf bytes.Buffer\n+\terr := handleHook([]string{data}, &buf, io.Discard)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, buf.String())\n+}\n+\n+// TestHandleHook_Failure covers every configured hook string. The CLI passes\n+// the matched hook config string as RootCmd, so multi-word hook entries such\n+// as \"buildx build\" arrive exactly as configured.\n+func TestHandleHook_Failure(t *testing.T) {\n+\ttests := []struct {\n+\t\tname string\n+\t\trootCmd string\n+\t\twantTmpl string\n+\t}{\n+\t\t{\n+\t\t\tname: \"build\",\n+\t\t\trootCmd: \"build\",\n+\t\t\twantTmpl: hintTemplates[\"build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"buildx build\",\n+\t\t\trootCmd: \"buildx build\",\n+\t\t\twantTmpl: hintTemplates[\"buildx build\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"run\",\n+\t\t\trootCmd: \"run\",\n+\t\t\twantTmpl: hintTemplates[\"run\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"compose\",\n+\t\t\trootCmd: \"compose\",\n+\t\t\twantTmpl: hintTemplates[\"compose\"],\n+\t\t},\n+\t\t{\n+\t\t\tname: \"unknown command falls back to default\",\n+\t\t\trootCmd: \"push\",\n+\t\t\twantTmpl: defaultHintTemplate,\n+\t\t},\n+\t}\n+\n+\tfor _, tc := range tests {\n+\t\tt.Run(tc.name, func(t *testing.T) {\n+\t\t\tdata := marshal(t, manager.HookPluginData{\n+\t\t\t\tRootCmd: tc.rootCmd,\n+\t\t\t\tCommandError: \"exit status 1\",\n+\t\t\t})\n+\n+\t\t\tvar buf bytes.Buffer\n+\t\t\terr := handleHook([]string{data}, &buf, io.Discard)\n+\t\t\trequire.NoError(t, err)\n+\n+\t\t\tvar msg hooks.HookMessage\n+\t\t\trequire.NoError(t, json.Unmarshal(buf.Bytes(), &msg))\n+\t\t\tassert.EqualValues(t, hooks.NextSteps, msg.Type)\n+\t\t\tassert.Equal(t, tc.wantTmpl, msg.Template)\n+\t\t})\n+\t}\n+}\n+\n+func TestHooks_SkippedWhenFeatureDisabled(t *testing.T) {\n+\torig := checkEnabled\n+\tcheckEnabled = func(context.Context) error {\n+\t\treturn errors.New(\"Docker AI is not enabled\")\n+\t}\n+\tt.Cleanup(func() { checkEnabled = orig })\n+\n+\t// The hook writes to the real process stdout by design, so capture it\n+\t// the same way cli/main_test.go's setStdout helper does: swap os.Stdout\n+\t// for a temp file around the command execution.\n+\ttmp, err := os.CreateTemp(t.TempDir(), \"stdout\")\n+\trequire.NoError(t, err)\n+\torigStdout := os.Stdout\n+\tos.Stdout = tmp\n+\tt.Cleanup(func() { os.Stdout = origStdout })\n+\n+\tdata := marshal(t, manager.HookPluginData{\n+\t\tRootCmd: \"build\",\n+\t\tCommandError: \"exit status 1\",\n+\t})\n+\n+\tcmd := Hooks()\n+\tcmd.SetArgs([]string{data})\n+\trequire.NoError(t, cmd.Execute())\n+\n+\trequire.NoError(t, tmp.Close())\n+\toutput, err := os.ReadFile(tmp.Name())\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, string(output), \"no hint should be shown when Docker AI is disabled\")\n+}\n+\n+func marshal(t *testing.T, v any) string {\n+\tt.Helper()\n+\tb, err := json.Marshal(v)\n+\trequire.NoError(t, err)\n+\treturn string(b)\n+}\n\ndiff --git a/cli/main.go b/cli/main.go\nindex be9cb6210..c57436bff 100644\n--- a/cli/main.go\n+++ b/cli/main.go\n@@ -43,6 +43,10 @@ func main() {\n \t\tcmd.AddCommand(commands.Thread())\n \t\tcmd.AddCommand(commands.Mcp())\n \n+\t\t// CLI hooks subcommand — registered for both V1 and V2 so that\n+\t\t// \"What's next:\" hints work regardless of the active feature flag.\n+\t\tcmd.AddCommand(commands.Hooks())\n+\n \t\toriginalPreRun := cmd.PersistentPreRunE\n \t\tcmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {\n \t\t\tif err := plugin.PersistentPreRunE(cmd, args); err != nil {\n\ndiff --git a/cli/main_test.go b/cli/main_test.go\nindex 2ccecd42f..338474e68 100644\n--- a/cli/main_test.go\n+++ b/cli/main_test.go\n@@ -18,6 +18,8 @@ func TestMain(m *testing.M) {\n \t// Make desktop.CheckFeatureIsEnabled a no-op so CLI tests never\n \t// require a running Docker Desktop.\n \tversion.GoTest = true\n+\t// The CLI hook tests below reuse this harness: with GoTest set, hook\n+\t// execution skips the Desktop check just like every other CLI test.\n \tos.Exit(m.Run())\n }\n \n@@ -98,6 +100,31 @@ func TestMCPBuiltin(t *testing.T) {\n \tassert.Contains(t, string(output), `\"docker\"`)\n }\n \n+func TestHooksShowsHintOnFailure(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"exit status 1\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Contains(t, string(output), `docker ai`)\n+\tassert.Contains(t, string(output), `build failure with Gordon`)\n+}\n+\n+func TestHooksNoOutputOnSuccess(t *testing.T) {\n+\ttmp := t.TempDir()\n+\tstdoutPath := setStdout(t, tmp)\n+\n+\trunDockerAI(\"docker-ai\", \"ai\", \"docker-cli-plugin-hooks\",\n+\t\t`{\"RootCmd\":\"build\",\"Flags\":{},\"CommandError\":\"\"}`)\n+\n+\toutput, err := os.ReadFile(stdoutPath)\n+\trequire.NoError(t, err)\n+\tassert.Empty(t, string(output))\n+}\n+\n func runDockerAI(args ...string) {\n \tos.Args = args\n \tmain()\n```", "created_at": "2026-02-18T11:16:21-05:00" } } diff --git a/review-pr/agents/pr-review.yaml b/review-pr/agents/pr-review.yaml index 0aaadcd..42450ac 100644 --- a/review-pr/agents/pr-review.yaml +++ b/review-pr/agents/pr-review.yaml @@ -195,12 +195,42 @@ agents: `{"findings": [], "summary": "Drafter did not complete", "review_complete": false}`. A `transfer_task` tool error counts as exactly this same case: if a drafter delegation fails or returns a tool error instead of output, treat that - delegation's response as the fallback object above — do NOT retry the delegation - and NEVER approve on the basis of an errored delegation. Likewise, a response + delegation's response as the fallback object above and NEVER approve on the + basis of an errored delegation. Do NOT retry it, with ONE narrow exception — + the bounded runtime-delegation retry defined below. Likewise, a response that parses as JSON but is partial, refused, or otherwise malformed (missing required fields, wrong types, or placeholder/refusal text instead of real content) must not be salvaged: treat it as the same fallback object so it follows the incomplete-review fallback below (`review_complete: false`). + **Bounded retry for runtime delegation failures (REQUIRED — runs after the + whole batch settles, BEFORE the aggregation rule below):** a delegation + error is a RUNTIME DELEGATION FAILURE only when the `transfer_task` tool + call itself failed with an error message showing the runtime rejected the + delegation before the drafter ran — e.g. containing "cannot transfer task" + or "target agent not in sub-agents list" (the known batched-transfer + misrouting failure). Drafter OUTPUT that is empty, malformed, refused, + partial, or schema-rejected is NEVER a runtime delegation failure and is + NEVER retried — it stays the fallback object above. After every batched + delegation has settled, collect the chunks that failed with a runtime + delegation failure and re-delegate each such chunk exactly ONCE, one at a + time (sequential single `transfer_task` calls — never re-batch retries), + keeping every successful delegation's response from the original batch. A + retry that succeeds contributes its parsed response exactly like a + first-attempt success; record the retry in that chunk's diagnostic summary + line (e.g. `chunk 3: (retried after runtime delegation failure)`). + A retry that fails — for ANY reason — becomes the fallback object above with + a summary line recording both attempts (e.g. `chunk 3: Drafter did not + complete (runtime delegation failure; sequential retry failed)`); never + retry it again. HARD BOUNDS: at most one retry per chunk, at most TWO + retried chunks per review (when more than two chunks failed this way, + retry the first two in chunk order and leave the rest as fallback + objects), and one retry pass per review. SKIP the retry pass entirely + when posting time is at risk — if the run is close to its budget (e.g. + verification and posting still pending on a large diff), leave every + failed chunk as its fallback object and post the incomplete review + instead: a posted incomplete review beats a timeout. A retry never turns + a failed or incomplete chunk into a success by itself — only a valid, + complete JSON response does. **Refusal content check (REQUIRED — applies to every parsed drafter response):** the schema only rejects empty strings (`minLength: 1`) — it cannot see whitespace-only or placeholder text. Inspect every @@ -214,16 +244,18 @@ agents: **Aggregation rule (REQUIRED — how per-delegation results become ONE merged result):** - Merged `findings` = the concatenation of every delegation's `findings` array. - Merged `review_complete` = true ONLY IF every delegation returned valid JSON - with `review_complete: true`. If ANY delegation errored, was malformed, - refused, or partial (i.e. became the fallback object above), or returned - `review_complete: false`, the merged `review_complete` is false. This holds + with `review_complete: true` (a chunk retried under the bounded + runtime-delegation retry counts by its final result). If ANY delegation + errored, was malformed, refused, or partial (i.e. became the fallback object + above), or returned `review_complete: false`, the merged `review_complete` is + false. This holds even when the merged findings list is empty — zero findings from an incomplete batch is NEVER grounds to approve. - Merged `summary` = a diagnostic aggregate with one line per delegation, labeled by its chunk (e.g. `chunk 3: `), so a crashed, refused, or truncated chunk stays visible in the posted review. Every later step reads these merged values. Check the merged `review_complete`: - - If `review_complete` is `true` AND zero findings → skip directly to step 8 (Decision Rules), then post a 🟢 APPROVE COMMENT review with an empty comments array. You MUST still post the review via `gh api` — do not exit without posting. + - If `review_complete` is `true` AND zero findings → skip directly to step 8 (Decision Rules), then post a 🟢 NO FINDINGS COMMENT review with an empty comments array. You MUST still post the review via `gh api` — do not exit without posting. - If `review_complete` is `false` AND zero findings → post a COMMENT review and do NOT approve. Use the incomplete-review body from Decision Rules rule 4 — the "### ⚠️ Review incomplete" heading, never an "### Assessment:" line. @@ -237,7 +269,7 @@ agents: from the chunks that DID complete, but carry the incompleteness through steps 8–9: the posted review MUST use the incomplete-review body from Decision Rules rule 4 (the "### ⚠️ Review incomplete" heading plus the - diagnostic merged `summary`) and MUST NOT carry a "🟢 APPROVE" label or any + diagnostic merged `summary`) and MUST NOT carry a "🟢 NO FINDINGS" label or any approve wording. Verified findings — however high their confidence — never overwrite incompleteness. - Otherwise: in GitHub posting mode FIRST apply the Prior Review Thread History @@ -247,7 +279,14 @@ agents: surviving findings with severity "high" or "medium" — reference each finding's one-line summary via the `issue` field and its explanation via the `details` field (not `title`/`body`) — and delegate them to the `verifier` - in a single batch. Skip verification for "low" findings. + in a single batch. Skip verification for "low" findings — but NEVER drop + them: every surviving "low" finding stays in the review as a summary-only + entry (step 9's "Low-severity findings (not verified, not posted inline)" + list) and blocks the 🟢 NO FINDINGS label like any other surviving finding + (Decision Rules rules 1–3). If NO surviving finding is "high" or "medium", + skip the verifier delegation entirely (there is nothing to verify) and + continue at step 8 with the surviving low findings — they are surfaced + and drive the assessment label; never approve over them. **Assign a `finding_id` before delegating**: number the delegated findings with sequential integers 1..N in the exact order they appear in the delegation message (first finding = 1) — deterministic, unique within the batch, assigned @@ -269,9 +308,17 @@ agents: fails or returns a tool error, which you MUST treat exactly like an empty/malformed response — post a COMMENT review that includes the drafter's unverified findings with a note that verification was inconclusive. + The inconclusive-verification body MUST open with "### ⚠️ Verification inconclusive" + — never an "### Assessment:" line and never approve wording ("### Assessment:" is + the completed-run marker the incremental reviewer keys on; an unverified review + must never carry it). Do NOT approve — surface the raw findings so the author can evaluate them. - Do NOT retry the delegation. A `transfer_task` tool error is never grounds to - approve or to retry. + Do NOT retry the delegation for empty, malformed, refused, or partial verifier + OUTPUT. ONE bounded exception, mirroring step 5: a runtime delegation failure + (the `transfer_task` call itself fails with e.g. "cannot transfer task" or + "target agent not in sub-agents list") may be retried exactly once, + sequentially; if the retry fails for any reason, apply this fallback — never a + second retry. A `transfer_task` tool error is never grounds to approve. (The fallback preserves the drafter's analysis for the author.) 6. Parse the verifier's JSON response (a `verdicts` array). **Refusal content check (REQUIRED — first, before the pairing check, scope @@ -323,6 +370,10 @@ agents: - **Lower-confidence summary** — non-forced findings scoring below the inline threshold T (plus any pushed past the comment cap), listed under "Lower-confidence findings (not posted inline)" with their scores. Never silently drop these. + - **Low-severity summary** — surviving low-severity findings (verification is + skipped for them), listed under "Low-severity findings (not verified, not posted + inline)" as `[low] file:line — issue`. Never silently drop these: each one blocks + the 🟢 NO FINDINGS label (Decision Rules rule 3). - **Dismissed security audit** — DISMISSED `security` findings, listed under "Dismissed security findings (review manually)" citing the verifier's stated mitigation. - **Incomplete-review notice** — when the merged `review_complete` (step 5) is false, @@ -345,7 +396,8 @@ agents: remaining (non-forced) inline comments, keep at most 5 (highest confidence first) and move the overflow to the lower-confidence summary list. - Find **real bugs in the changed code**, not style issues. If the changed code works correctly, approve it. + Find **real bugs in the changed code**, not style issues. If the changed code works + correctly, report zero findings — never dress the outcome up as an approval. ## CRITICAL: Only flag problems this PR introduces @@ -473,11 +525,18 @@ agents: Some repos lack branch protection; `APPROVE` would bypass human review, `REQUEST_CHANGES` would block merging. The bot provides feedback only. - **Zero-findings posting** (use only when findings are empty AND the merged - `review_complete` is true — incomplete reviews must instead post the - "### ⚠️ Review incomplete" body from Decision Rules rule 4, never a 🟢 APPROVE body): - set `REVIEW_BODY="### Assessment: 🟢 APPROVE"` and use the rendered command in - `/tmp/refs/posting-format.md`. It already contains the validated immutable `commit_id`. + **Zero-findings posting** (use only when ZERO findings of ANY severity survive — + none inline AND none in any summary list, low included — AND the merged + `review_complete` is true AND verification, where required, was conclusive. + Incomplete reviews must instead post the "### ⚠️ Review incomplete" body from + Decision Rules rule 4, and inconclusive verification the + "### ⚠️ Verification inconclusive" body — never a 🟢 NO FINDINGS body): + write `### Assessment: 🟢 NO FINDINGS` to `/tmp/review_body.md` (quoted heredoc), + stage the empty comments array (`echo '[]' > /tmp/review_comments.json` — the + validator refuses a NO FINDINGS body over ANY staged comment), and run the + rendered posting chain in `/tmp/refs/posting-format.md` exactly as + rendered. It already contains the validated immutable `commit_id`, the trusted + repository/PR review route, and the trusted body-validation step. This call is mandatory even when there are zero findings. - **Console output mode**: Output markdown (see Console format below). Never call `gh api`. @@ -585,17 +644,34 @@ agents: ## Decision Rules (MANDATORY — strict lookup, not a judgment call) - 1. **Filter**: Consider only findings whose confidence disposition is `inline` (see - Confidence Scoring). Out-of-scope, dropped, summary-only, and audit-only findings do - NOT drive the assessment label. - 2. **Classify** the inline findings (for informational labeling in the review summary): + YOU apply these rules — nothing else recomputes the assessment at runtime. + `src/review-assessment/review-assessment.ts` is their executable spec: an + outcome-for-outcome mirror pinned by unit tests, not a runtime enforcer. + Change one, change both — the tests pin every outcome. + + 1. **Collect the SURVIVING findings**: a finding survives when it is in scope and + was not dismissed or dropped — i.e. it will be surfaced anywhere in the review: + inline comments, the lower-confidence summary, the medium-severity floor list, + and the unverified low-severity list (low findings skip verification but still + survive). Out-of-scope, dropped (negligible-band low), DISMISSED, and + audit-only (dismissed security) findings do not survive and do NOT drive the + assessment label. + 2. **Classify** the surviving findings (for informational labeling in the review summary): - CRITICAL = high severity CONFIRMED/LIKELY - NOTABLE = medium severity CONFIRMED/LIKELY - - MINOR = everything else + - MINOR = every other surviving finding (summary-only entries and unverified + low-severity findings included) 3. **Label the assessment** (informational only — does NOT change the event type): - ANY CRITICAL findings → label as "🔴 CRITICAL" in the summary - - ANY NOTABLE findings (no CRITICAL) → label as "🟡 NEEDS ATTENTION" - - Only MINOR or no findings → label as "🟢 APPROVE" + - ANY other surviving finding (NOTABLE or MINOR — any severity, any surfaced + disposition) → label as "🟡 NEEDS ATTENTION" + - EXACTLY ZERO surviving findings → label as "🟢 NO FINDINGS" + "🟢 NO FINDINGS" is emitted ONLY for a complete review (merged `review_complete` + true) with conclusive verification and zero surviving findings of EVERY + severity. A review that surfaces ANY finding — inline or summary-only, + including a single unverified low — must NOT carry the NO FINDINGS label. + The label is a neutral completion marker, NEVER an approval: the bot never + approves a PR, so no body may say "APPROVE", "LGTM", or "No issues found". 4. **Incompleteness override (fail-closed)**: if the merged `review_complete` from step 5 is false, the review is incomplete at ANY finding count — a crashed, refused, or truncated chunk means part of the diff was never reviewed — so @@ -607,15 +683,19 @@ agents: - Open the review body with "### ⚠️ Review incomplete" followed by the diagnostic merged `summary` (the per-chunk status lines from step 5). - Add a "Findings so far:" line under that heading: "🔴 CRITICAL" or - "🟡 NEEDS ATTENTION" when rule 3 produced those; a would-be "🟢 APPROVE" + "🟡 NEEDS ATTENTION" when rule 3 produced those; a would-be "🟢 NO FINDINGS" becomes "⚠️ INCOMPLETE" instead. - Inline comments from the completed chunks are still posted (steps 9–10 unchanged). Confidence scores decide per-finding dispositions only — they never restore a complete or approving outcome. + - This override outranks the inconclusive-verification fallback (step 5's + ANTI-LOOP rule): when the review is BOTH incomplete and unverified, the + body opens with "### ⚠️ Review incomplete" and notes the inconclusive + verification inside — neither body ever carries "### Assessment:". 5. **Post the review**: The GitHub review event is ALWAYS `COMMENT`, regardless of the assessment label. Never use `APPROVE` or `REQUEST_CHANGES`. This applies even when the findings list is empty — the review body must - still be posted: with the 🟢 APPROVE assessment label when the merged + still be posted: with the 🟢 NO FINDINGS assessment label when the merged `review_complete` is true, or with the rule-4 "### ⚠️ Review incomplete" body when it is false. @@ -626,8 +706,16 @@ agents: Use `jq` (never raw `echo`) to build JSON. Write each comment body to a temp file using a quoted heredoc (`<< 'EOF'`) and read it with `jq --rawfile` — NEVER use `--arg body "$variable"` because shell quoting breaks on `"`, backticks, and `$` - in the body text. Each finding becomes an inline comment with `` - marker on its own line. Do NOT include the marker in console mode. + in the body text. The REVIEW body follows the same contract: write it to + `/tmp/review_body.md` via a quoted heredoc and post with the rendered chain in + the template exactly as staged — its trusted validation step (`test -s` + + `node /tmp/review-assessment.js finalize-body …`) refuses a body without exactly + one valid status line, refuses a missing or non-array `/tmp/review_comments.json` + (and a 🟢 NO FINDINGS body over ANY staged inline comment), and appends the + run's attribution marker; a hand-rolled `gh api` call skips that marker and the + workflow then reports the run as unverified. Each finding becomes an inline + comment with `` marker on its own line. Do NOT + include the marker in console mode. ## Suggestion blocks (GitHub posting mode) @@ -680,7 +768,7 @@ agents: ``` ## Review: COMMENT - ### Assessment: [🟢 APPROVE|🟡 NEEDS ATTENTION|🔴 CRITICAL] + ### Assessment: [🟢 NO FINDINGS|🟡 NEEDS ATTENTION|🔴 CRITICAL] ### Findings **[SEVERITY] file:line — issue** (confidence: BAND SCORE/100) details @@ -688,10 +776,14 @@ agents: ### Lower-confidence findings (not posted inline) - [SEVERITY] file:line — issue (confidence: BAND SCORE/100) + ### Low-severity findings (not verified, not posted inline) + - [low] file:line — issue + ### Dismissed security findings (review manually) - file:line — issue (verifier mitigation: …) ``` - Omit the "Lower-confidence" and "Dismissed security" sections when they have no entries. + Omit the "Lower-confidence", "Low-severity", and "Dismissed security" sections when + they have no entries. When the merged `review_complete` is false — at ANY finding count — replace the "### Assessment:" line with the incomplete-review header from Decision Rules rule 4 and never print an approve label: diff --git a/review-pr/agents/refs/posting-format.md b/review-pr/agents/refs/posting-format.md index cb947b0..4cfcc0c 100644 --- a/review-pr/agents/refs/posting-format.md +++ b/review-pr/agents/refs/posting-format.md @@ -27,17 +27,41 @@ with `echo` — this causes double-escaping of newlines (`\n` rendered as litera Build the review body and comments, then use `jq` to produce correctly-escaped JSON: ```bash -# Review body is the assessment badge, plus the lower-confidence and dismissed-security -# summary sections when they have entries (high-confidence findings go in inline comments). -# Append each section only when non-empty, e.g.: +# Review body: write it to /tmp/review_body.md via a QUOTED heredoc — never a +# shell variable (quoting breaks on ", backticks, and $) and never a default +# copied from this file. The body is the header you computed via the Decision +# Rules, plus the lower-confidence, low-severity, and dismissed-security +# summary sections when they have entries (high-confidence findings go in +# inline comments). Exactly ONE status line, chosen by YOUR computed outcome: +# - incomplete review (merged review_complete false) +# → body opens "### ⚠️ Review incomplete" (never an "### Assessment:" line) +# - verification inconclusive (malformed/unpaired verifier batch) +# → body opens "### ⚠️ Verification inconclusive" (never "### Assessment:") +# - ANY surviving finding — inline or summary-only, low severity included +# → "### Assessment: 🔴 CRITICAL" or "### Assessment: 🟡 NEEDS ATTENTION" +# - complete review, conclusive verification, ZERO surviving findings of every +# severity → "### Assessment: 🟢 NO FINDINGS" (the ONLY zero-findings outcome — +# a neutral completion label; the bot never approves, so never write APPROVE, +# LGTM, or "No issues found" wording into the body) +# Legitimate note text (e.g. the incremental-review coverage note) may precede +# the single status line. The validation step below refuses to post a body +# that is empty, has no/conflicting status lines, carries approve/LGTM +# wording, or pairs 🟢 NO FINDINGS with any findings section. +cat > /tmp/review_body.md << 'REVIEW_BODY_EOF' + +REVIEW_BODY_EOF +# Example shape of a completed body with summary sections: # ### Assessment: 🟡 NEEDS ATTENTION # # #### Lower-confidence findings (not posted inline) # - [medium] file.go:42 — issue (confidence: weak 48/100) # +# #### Low-severity findings (not verified, not posted inline) +# - [low] file.go:12 — issue +# # #### Dismissed security findings (review manually) # - file.go:88 — issue (verifier mitigation: …) -REVIEW_BODY="### Assessment: 🟢 APPROVE" # or 🟡 NEEDS ATTENTION / 🔴 CRITICAL # Start with an empty comments array echo '[]' > /tmp/review_comments.json @@ -110,15 +134,29 @@ jq '[.[] | select(.body | length > 0)]' /tmp/review_comments.json > /tmp/review_ && mv /tmp/review_comments.tmp /tmp/review_comments.json echo "Posting review with $(jq length /tmp/review_comments.json) inline comment(s)" -# The composite action replaces __PR_HEAD_SHA__ with the validated immutable review snapshot -# before the agent runs. This command must contain the selected literal SHA. -jq -n \ - --arg body "$REVIEW_BODY" \ - --arg event "COMMENT" \ - --arg commit_id "__PR_HEAD_SHA__" \ - --slurpfile comments /tmp/review_comments.json \ - '{body: $body, event: $event, commit_id: $commit_id, comments: $comments[0]}' \ -| gh api repos/{owner}/{repo}/pulls/{pr}/reviews --input - +# The composite action replaces __PR_HEAD_SHA__ with the validated immutable review +# snapshot, __REPOSITORY__/__PR_NUMBER__ with the trusted repository and PR number, +# and __REVIEW_RUN_NONCE__ with this run's attribution nonce before the agent runs. +# Run the chained command exactly as rendered — never rewrite the route, substitute +# owner/repo/PR values, or skip the validation steps. The chain refuses to post when +# the body file is missing/empty, when /tmp/review_comments.json is missing or not a +# JSON array, when a 🟢 NO FINDINGS body is paired with ANY staged inline comment, +# or when the trusted validator rejects the body; the validator also appends this +# run's hidden attribution marker, which the workflow requires to verify that the +# review was actually posted — a bypassed or hand-rolled posting command is reported +# as an unverified run. The payload is staged to a trusted temp file and checked +# before posting, so `gh api` is never invoked when jq fails to construct it. +test -s /tmp/review_body.md \ + && node /tmp/review-assessment.js finalize-body /tmp/review_body.md __REVIEW_RUN_NONCE__ /tmp/review_comments.json \ + && jq -n \ + --rawfile body /tmp/review_body.md \ + --arg event "COMMENT" \ + --arg commit_id "__PR_HEAD_SHA__" \ + --slurpfile comments /tmp/review_comments.json \ + '{body: $body, event: $event, commit_id: $commit_id, comments: $comments[0]}' \ + > /tmp/review_payload.json \ + && jq -e 'type == "object"' /tmp/review_payload.json > /dev/null \ + && gh api "repos/__REPOSITORY__/pulls/__PR_NUMBER__/reviews" --input - < /tmp/review_payload.json ``` The `` marker MUST be on its own line, separated by a blank line diff --git a/src/incremental-review/__tests__/incremental-review.test.ts b/src/incremental-review/__tests__/incremental-review.test.ts index ebc00c7..de8b5f8 100644 --- a/src/incremental-review/__tests__/incremental-review.test.ts +++ b/src/incremental-review/__tests__/incremental-review.test.ts @@ -28,7 +28,7 @@ const HEAD_SHA = 'c'.repeat(40); function review(overrides: Partial = {}): ReviewLike { return { user: { login: 'docker-agent' }, - body: '### Assessment: 🟢 APPROVE', + body: '### Assessment: 🟢 NO FINDINGS', commit_id: SHA_A, submitted_at: '2026-01-01T10:00:00Z', ...overrides, @@ -53,8 +53,68 @@ describe('findLastReviewedSha', () => { expect(findLastReviewedSha(reviews)).toBe(SHA_B); }); - it('accepts the LGTM fallback body as a completed review', () => { - expect(findLastReviewedSha([review({ body: '🟢 **No issues found** — LGTM!' })])).toBe(SHA_A); + it('rejects the legacy LGTM fallback body (synthesized from exit 0, never a checkpoint)', () => { + expect(findLastReviewedSha([review({ body: '🟢 **No issues found** — LGTM!' })])).toBeNull(); + }); + + it('still checkpoints the LEGACY "### Assessment: 🟢 APPROVE" body', () => { + // Reviews posted before the zero-findings label was neutralized to + // "🟢 NO FINDINGS" are genuine completed runs: the "### Assessment:" + // marker — not the label — is what advances the checkpoint. + expect(findLastReviewedSha([review({ body: '### Assessment: 🟢 APPROVE' })])).toBe(SHA_A); + }); + + it('never advances to a newer legacy LGTM SHA over an older valid assessment', () => { + // Defense-in-depth for historical false LGTMs: the newest entry being a + // legacy LGTM must not shadow the older genuinely-completed review — the + // commits after SHA_A were never reviewed and must be re-covered. + const reviews = [ + review({ commit_id: SHA_A, submitted_at: '2026-01-01T10:00:00Z' }), + review({ + body: '🟢 **No issues found** — LGTM!', + commit_id: SHA_B, + submitted_at: '2026-01-03T10:00:00Z', + }), + ]; + expect(findLastReviewedSha(reviews)).toBe(SHA_A); + }); + + it('lets a newer valid assessment win over older legacy LGTM and fallback bodies', () => { + // Ignoring legacy LGTMs must not pin the checkpoint in the past forever: + // once a later real completed review exists, its SHA wins. + const reviews = [ + review({ + body: '🟢 **No issues found** — LGTM!', + commit_id: SHA_A, + submitted_at: '2026-01-01T10:00:00Z', + }), + review({ + body: '⚠️ **Review incomplete** — no review was posted.', + commit_id: SHA_A, + submitted_at: '2026-01-02T10:00:00Z', + }), + review({ commit_id: SHA_B, submitted_at: '2026-01-03T10:00:00Z' }), + ]; + expect(findLastReviewedSha(reviews)).toBe(SHA_B); + }); + + it('rejects incomplete and inconclusive fallback bodies', () => { + const bodies = [ + '⚠️ **Review incomplete** — The review agent finished without posting a review.', + '### ⚠️ Review incomplete\nchunk 2: Drafter did not complete', + '### ⚠️ Verification inconclusive\nUnverified findings below.', + ]; + for (const body of bodies) { + expect(findLastReviewedSha([review({ body })]), body).toBeNull(); + } + }); + + it('never checkpoints a body combining an assessment line with an incomplete marker', () => { + expect( + findLastReviewedSha([ + review({ body: '### ⚠️ Review incomplete\n### Assessment: 🟢 NO FINDINGS' }), + ]), + ).toBeNull(); }); it('accepts the GitHub App bot login variant', () => { @@ -65,6 +125,53 @@ describe('findLastReviewedSha', () => { expect(findLastReviewedSha([review({ user: { login: 'alice' } })])).toBeNull(); }); + it('checkpoints marker-bearing action reviews posted by other [bot] identities', () => { + // The action's github-token input defaults to github.token, which posts + // as github-actions[bot] — those runs embed the per-run attribution + // marker, so their completed reviews still advance the checkpoint. + const marker = ``; + const marked = review({ + user: { login: 'github-actions[bot]' }, + body: `### Assessment: 🟡 NEEDS ATTENTION\n\n${marker}`, + }); + expect(findLastReviewedSha([marked])).toBe(SHA_A); + }); + + it('never checkpoints a marker-bearing review from a non-[bot] login', () => { + // The marker format is public, so a PR author could paste one into their + // own review to pin the checkpoint past unreviewed commits. Only + // App-reserved [bot] logins (or the legacy docker-agent identities) count. + const marker = ``; + const forged = review({ + user: { login: 'mallory' }, + body: `### Assessment: 🟢 NO FINDINGS\n\n${marker}`, + }); + expect(findLastReviewedSha([forged])).toBeNull(); + }); + + it('never checkpoints an unmarked review from a non-legacy [bot] login', () => { + const unmarked = review({ + user: { login: 'github-actions[bot]' }, + body: '### Assessment: 🟢 NO FINDINGS', + }); + expect(findLastReviewedSha([unmarked])).toBeNull(); + }); + + it('rejects marker-bearing incomplete/inconclusive bodies from [bot] identities', () => { + const marker = ``; + const bodies = [ + `### ⚠️ Review incomplete\nchunk 2: Drafter did not complete\n\n${marker}`, + `### ⚠️ Verification inconclusive\nUnverified findings below.\n\n${marker}`, + `⚠️ **Review incomplete** — no review was posted.\n\n${marker}`, + ]; + for (const body of bodies) { + expect( + findLastReviewedSha([review({ user: { login: 'github-actions[bot]' }, body })]), + body, + ).toBeNull(); + } + }); + it('ignores timeout and failure fallback reviews (commits stay unreviewed)', () => { const reviews = [ review({ body: '⏱️ **PR Review Timed Out** — retry.' }), diff --git a/src/incremental-review/__tests__/index.test.ts b/src/incremental-review/__tests__/index.test.ts index 08ccfe3..c66074f 100644 --- a/src/incremental-review/__tests__/index.test.ts +++ b/src/incremental-review/__tests__/index.test.ts @@ -76,7 +76,7 @@ const INCREMENTAL_DIFF = [ function completedReview(): ReviewLike { return { user: { login: 'docker-agent' }, - body: '### Assessment: 🟢 APPROVE', + body: '### Assessment: 🟢 NO FINDINGS', commit_id: SHA_A, submitted_at: '2026-01-01T10:00:00Z', }; diff --git a/src/incremental-review/incremental-review.ts b/src/incremental-review/incremental-review.ts index 63e4c34..2ab9e73 100644 --- a/src/incremental-review/incremental-review.ts +++ b/src/incremental-review/incremental-review.ts @@ -12,10 +12,12 @@ * every posted review: `commit_id` on `GET /pulls/{n}/reviews` is the PR head * SHA at posting time. This survives across workflow runs, requires no extra * writes, and cannot be edited away like a marker embedded in a comment body. - * Only reviews that represent a *completed* run count — an assessment body - * ("### Assessment:") or the zero-findings LGTM fallback. Timeout and failure - * fallback reviews do NOT mark commits as reviewed, so the next run re-covers - * them. + * Only reviews that represent a *completed* run count — a body carrying the + * "### Assessment:" header and no incomplete/inconclusive marker. Timeout, + * failure, and incomplete/no-post fallback reviews do NOT mark commits as + * reviewed, so the next run re-covers them. The legacy zero-findings LGTM + * fallback ("🟢 **No issues found**") is deliberately not trusted either: it + * was synthesized from exit code 0 alone, without evidence a review happened. * * ## Fallbacks to a full review (see planIncrementalReview) * @@ -35,6 +37,8 @@ * PR diff, so GitHub would reject inline comments anchored there (HTTP 422). */ +import { isActionPostedReview } from '../review-assessment/review-assessment.js'; + /** Result of one git invocation. Injectable for deterministic tests. */ export interface GitResult { ok: boolean; @@ -64,17 +68,33 @@ export interface IncrementalPlan { // looser (e.g. a body that starts with "-") must never get through. const SHA40 = /^[0-9a-f]{40}$/i; -// Bodies that mark a review run as completed. The timeout ("⏱️ **PR Review -// Timed Out**") and failure ("❌ **PR Review Failed**") fallbacks match -// neither, so unreviewed commits stay unreviewed. -const COMPLETED_BODY_MARKERS = ['### Assessment:', '🟢 **No issues found**']; +// Bodies that mark a review run as completed: only the assessment header a +// finished pipeline emits. The timeout ("⏱️ **PR Review Timed Out**"), failure +// ("❌ **PR Review Failed**"), and no-post ("⚠️ **Review incomplete**") +// fallbacks match nothing here, so unreviewed commits stay unreviewed. The +// legacy zero-findings LGTM fallback ("🟢 **No issues found**") is NOT a +// completion marker: it was posted blindly on exit 0 without a posted-review +// marker, so trusting it would permanently skip commits that were never +// actually reviewed. +const COMPLETED_BODY_MARKERS = ['### Assessment:']; + +// Bodies that mark a review run as NOT completed, whatever else they contain. +// Defense-in-depth: an incomplete or inconclusive review must never advance +// the checkpoint even if an "### Assessment:" line leaks into the same body. +const INCOMPLETE_BODY_MARKERS = [ + '### ⚠️ Review incomplete', + '### ⚠️ Verification inconclusive', + '⚠️ **Review incomplete**', +]; // GitHub presents the bot identity as "docker-agent" when posting with a // machine user token, or "docker-agent[bot]" through a GitHub App installation -// token. Match both (same convention as src/rate-limit). -function matchesBotLogin(login: string | null | undefined, botLogin: string): boolean { - return login === botLogin || login === `${botLogin}[bot]`; -} +// token. The action's public github-token input additionally lets consumers +// post through other identities (the default github.token posts as +// "github-actions[bot]"), so reviews carrying the action's per-run marker are +// also recognized when their login is `[bot]`-suffixed — see +// isActionPostedReview for why plain user logins never qualify (a forged +// marker must not pin the checkpoint past unreviewed commits). /** * Find the head SHA recorded on the most recent *completed* docker-agent @@ -86,9 +106,10 @@ export function findLastReviewedSha( ): string | null { let best: { sha: string; at: number } | null = null; for (const review of reviews) { - if (!matchesBotLogin(review.user?.login, botLogin)) continue; + if (!isActionPostedReview(review.user?.login, review.body, botLogin)) continue; const body = review.body ?? ''; if (!COMPLETED_BODY_MARKERS.some((marker) => body.includes(marker))) continue; + if (INCOMPLETE_BODY_MARKERS.some((marker) => body.includes(marker))) continue; const sha = review.commit_id ?? ''; if (!SHA40.test(sha)) continue; if (!review.submitted_at) continue; diff --git a/src/pr-review-agent/__tests__/eval-fixtures.test.ts b/src/pr-review-agent/__tests__/eval-fixtures.test.ts new file mode 100644 index 0000000..44af175 --- /dev/null +++ b/src/pr-review-agent/__tests__/eval-fixtures.test.ts @@ -0,0 +1,273 @@ +// Copyright The Docker Agent Action authors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Deterministic validation of the model-eval fixtures in review-pr/agents/evals/. + * + * The docker-agent eval runner (pkg/evaluation in docker/docker-agent) imposes + * hard environment constraints that repeatedly produced broken fixtures: + * + * - Per-eval criteria accept ONLY `relevance`, `working_dir`, `size`, + * `setup`, and `image` (EvalCriteria unmarshals with + * DisallowUnknownFields). There is NO per-eval env field. + * - The container runs `sh /setup.sh && exec /docker-agent run …`: setup is + * a CHILD shell, so `export GITHUB_ACTIONS=true` never reaches the agent + * process. Every eval therefore runs in console output mode; a fixture + * asserting GitHub posting mode can never pass. + * - Some runner versions treat ANY setup stderr as fatal even on exit 0. + * Alpine's `apk add nodejs` prints an ICU packaging note to stderr, so + * fixtures must not install Node. + * - The eval image contains only the mounted agents dir (/configs) and an + * empty /working_dir — the repo's gitignored dist/ bundles do not exist, + * so `node dist/….js` in setup is always MODULE_NOT_FOUND. + * - Evals must never be able to write to real GitHub repositories. + * + * These tests pin those invariants for every fixture, plus fixture-specific + * contracts (the success trio is one repeated payload; the marlin + * event-firing eval keeps its two essential bugs without the retired + * redundant duplicate-timestamp criterion). + */ +import { readdirSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const EVALS_DIR = resolve(import.meta.dirname, '../../../review-pr/agents/evals'); +const POSTING_TEMPLATE_PATH = resolve( + import.meta.dirname, + '../../../review-pr/agents/refs/posting-format.md', +); + +/** Keys session.EvalCriteria accepts — unknown keys fail the eval run. */ +const SUPPORTED_EVAL_KEYS = ['relevance', 'working_dir', 'size', 'setup', 'image']; + +interface Fixture { + name: string; + raw: string; + id: string; + title: string; + evals: Record; + relevance: string[]; + setup: string; + userContents: string[]; +} + +function asRecord(value: unknown, context: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`${context} is not an object`); + } + return value as Record; +} + +function loadFixture(name: string): Fixture { + const raw = readFileSync(resolve(EVALS_DIR, name), 'utf-8'); + const doc = asRecord(JSON.parse(raw), name); + const evals = asRecord(doc.evals, `${name} evals`); + const relevance = Array.isArray(evals.relevance) + ? evals.relevance.filter((entry): entry is string => typeof entry === 'string') + : []; + const messages = Array.isArray(doc.messages) ? doc.messages : []; + const userContents: string[] = []; + for (const item of messages) { + const inner = asRecord(asRecord(item, `${name} messages[]`).message, `${name} message`); + const message = asRecord(inner.message, `${name} message.message`); + if (message.role === 'user' && typeof message.content === 'string') { + userContents.push(message.content); + } + } + return { + name, + raw, + id: typeof doc.id === 'string' ? doc.id : '', + title: typeof doc.title === 'string' ? doc.title : '', + evals, + relevance, + setup: typeof evals.setup === 'string' ? evals.setup : '', + userContents, + }; +} + +const fixtureNames = readdirSync(EVALS_DIR) + .filter((name) => name.endsWith('.json')) + .sort(); +const fixtures = fixtureNames.map(loadFixture); +const byName = new Map(fixtures.map((fixture) => [fixture.name, fixture])); + +function mustGet(name: string): Fixture { + const fixture = byName.get(name); + if (!fixture) throw new Error(`expected fixture missing: ${name}`); + return fixture; +} + +describe('eval fixture schema', () => { + it('finds the eval fixtures', () => { + expect(fixtureNames.length).toBeGreaterThan(0); + }); + + it.each(fixtureNames)('%s parses with a valid basic shape', (name) => { + const fixture = mustGet(name); + expect(fixture.id).not.toBe(''); + expect(fixture.title).not.toBe(''); + expect(fixture.relevance.length).toBeGreaterThan(0); + for (const criterion of fixture.relevance) { + expect(criterion.trim()).not.toBe(''); + } + expect(fixture.userContents.length).toBeGreaterThan(0); + for (const content of fixture.userContents) { + expect(content.trim()).not.toBe(''); + } + }); + + it.each(fixtureNames)('%s only uses eval keys the runner accepts', (name) => { + // EvalCriteria rejects unknown fields, so a typo (or an unsupported + // field like `env`) fails the whole eval run at load time. + const unknown = Object.keys(mustGet(name).evals).filter( + (key) => !SUPPORTED_EVAL_KEYS.includes(key), + ); + expect(unknown).toEqual([]); + }); +}); + +describe('console-mode honesty (no per-eval env exists)', () => { + it.each(fixtureNames)('%s does not export GITHUB_ACTIONS from setup', (name) => { + // Setup runs in a child shell; exported variables never reach the agent + // process. A fixture relying on this export tests nothing. + expect(mustGet(name).setup).not.toMatch(/GITHUB_ACTIONS\s*=/); + }); + + it.each(fixtureNames)('%s does not assert GitHub posting mode in relevance', (name) => { + for (const criterion of mustGet(name).relevance) { + expect(criterion).not.toMatch(/GITHUB_ACTIONS=true/); + expect(criterion.toLowerCase()).not.toContain('posting mode'); + } + }); +}); + +describe('setup environment constraints', () => { + it.each(fixtureNames)('%s never posts to GitHub from setup', (name) => { + const { setup } = mustGet(name); + // Installing github-cli is fine; invoking gh (or pushing) is not. + expect(setup).not.toMatch(/\bgh\s+(api|pr|repo)\b/); + expect(setup).not.toMatch(/\bgit\s+push\b/); + expect(setup).not.toMatch(/\bcurl\b/); + }); + + it.each(fixtureNames)('%s does not depend on Node in the eval container', (name) => { + const { setup } = mustGet(name); + // dist/ bundles are gitignored and never staged into the eval image, and + // Alpine's nodejs package prints an ICU note to stderr that some runner + // versions treat as a fatal setup failure despite exit 0. + expect(setup).not.toMatch(/\bnode\s/); + expect(setup).not.toMatch(/apk\s+add[^&|;]*\bnodejs\b/); + }); + + it.each(fixtureNames)('%s renders every posting-format placeholder it stages', (name) => { + const { setup } = mustGet(name); + if (!setup.includes('posting-format.md')) return; + const template = readFileSync(POSTING_TEMPLATE_PATH, 'utf-8'); + const placeholders = new Set(template.match(/__[A-Z_]+__/g) ?? []); + expect(placeholders.size).toBeGreaterThan(0); + for (const placeholder of placeholders) { + // Each placeholder must be substituted by the staging sed program + // (any delimiter); otherwise the agent reads a template with literal + // __X__ markers. + expect(setup, `${name} setup must substitute ${placeholder}`).toMatch( + new RegExp(`s[/|#]${placeholder}[/|#]`), + ); + } + }); +}); + +describe('success clean-control trio', () => { + const runNames = ['success-1.json', 'success-2.json', 'success-3.json']; + + it('keeps all three runs present', () => { + for (const name of runNames) expect(byName.has(name)).toBe(true); + }); + + it('repeats one identical payload, varying only id and run-numbered title', () => { + const [first, second, third] = runNames.map(mustGet); + const payload = (fixture: Fixture) => { + const doc = asRecord(JSON.parse(fixture.raw), fixture.name); + delete doc.id; + delete doc.title; + return JSON.stringify(doc); + }; + expect(payload(second)).toBe(payload(first)); + expect(payload(third)).toBe(payload(first)); + const ids = new Set(runNames.map((name) => mustGet(name).id)); + expect(ids.size).toBe(3); + runNames.forEach((name, index) => { + expect(mustGet(name).title).toMatch(new RegExp(`\\(run ${index + 1}\\)$`)); + }); + }); + + it('embeds the authoritative context that retired each historical false positive', () => { + const [content] = mustGet('success-1.json').userContents; + // RootCmd is the matched hook config string, so "buildx build" is a live + // map entry (docker/cli#6794) … + expect(content).toContain('docker/cli#6794'); + expect(content).toContain('"buildx build": `Debug this build failure with Gordon'); + expect(content).toContain('never just the top-level command token'); + // … TestMain sets version.GoTest so tests need no Docker Desktop … + expect(content).toContain('version.GoTest = true'); + expect(content).toContain('never need a running Docker Desktop'); + // … and writing to the process streams is deliberate and test-covered. + expect(content).toContain('return handleHook(args, os.Stdout, os.Stderr)'); + expect(content).toContain('Hook output deliberately goes to the process streams'); + expect(content).toContain('stdoutPath := setStdout(t, tmp)'); + }); + + it('requires neutral comment-only semantics and no surviving high findings', () => { + const { relevance } = mustGet('success-1.json'); + const joined = relevance.join('\n'); + expect(joined).toContain("it is NOT '🔴 CRITICAL'"); + expect(joined).toContain( + "No findings have severity 'high' with verdict 'CONFIRMED' or 'LIKELY'", + ); + expect(joined).toContain('neutral comment-only semantics'); + expect(joined).toContain("does not use approval wording such as 'APPROVE' or 'LGTM'"); + // The retired 🟢 APPROVE label must not resurface in criteria. + expect(joined).not.toContain('🟢 APPROVE'); + }); +}); + +describe('marlin event-firing criteria', () => { + it('keeps both essential bugs and drops the redundant duplicate-timestamp criterion', () => { + const { relevance } = mustGet('marlin-event-firing-react-1.json'); + const joined = relevance.join('\n'); + // Essential bug 1: track() runs in the render body, firing every render. + expect(joined).toContain('render body, not inside a useEffect'); + // Essential bug 2: Date.now() is the wrong type for a Timestamp field. + expect(joined).toContain('wrong type for a Timestamp field'); + // Retired: a third criterion re-demanding Date.now() duplicate-timestamp + // phrasing was redundant with the two above and failed correct reviews. + expect(joined.toLowerCase()).not.toContain('duplicate timestamps'); + }); +}); + +describe('console-mode git fixtures', () => { + const gitFixtures = ['large-diff-chunking-1.json', 'auto-filter-integration-1.json']; + + it.each(gitFixtures)('%s builds a local git repo so console mode has a real diff', (name) => { + const { setup } = mustGet(name); + // The console flow diffs merge-base(main, HEAD)..HEAD; setup must create + // both refs and must silence benign git stderr (quiet flags) because + // some runner versions treat any setup stderr as fatal. + expect(setup).toContain('git init -q -b main'); + expect(setup).toContain('git checkout -q -b'); + expect(setup).toMatch(/git commit -q/); + expect(setup).toContain('set -eu'); + }); + + it('keeps the SQL-injection signal and console expectations in both fixtures', () => { + for (const name of gitFixtures) { + const { relevance, setup } = mustGet(name); + const joined = relevance.join('\n'); + expect(setup).toContain('fmt.Sprintf("SELECT'); + expect(joined).toContain('SQL injection'); + expect(joined).toContain('pkg/storage/db.go'); + expect(joined).toContain("severity 'high'"); + expect(joined).toContain('console'); + } + }); +}); diff --git a/src/pr-review-agent/__tests__/pr-review-yaml.test.ts b/src/pr-review-agent/__tests__/pr-review-yaml.test.ts index 91143ad..b175422 100644 --- a/src/pr-review-agent/__tests__/pr-review-yaml.test.ts +++ b/src/pr-review-agent/__tests__/pr-review-yaml.test.ts @@ -26,8 +26,10 @@ * object, `review_complete: false`, no salvage) or the WHOLE verifier * batch (inconclusive COMMENT fallback before pairing — no approve, no * retry, no partial merge). The root must also treat a `transfer_task` - * tool error exactly like empty/malformed output (never approve, never - * retry), aggregate batched CI drafter responses fail-closed (merged + * tool error exactly like empty/malformed output (never approve; never + * retried except under the bounded runtime-delegation retry, which is + * recognized-failure-only, sequential, and once per chunk), aggregate + * batched CI drafter responses fail-closed (merged * review_complete is true only when EVERY delegation returned valid JSON * with review_complete true; an incomplete merge never approves at ANY * finding count and must post an explicit incomplete heading carrying the @@ -38,6 +40,15 @@ * COMMENT fallback, no partial merge — JSON Schema cannot express this * cardinality, so it lives in the instructions). * + * Honest limitation: layer 2 — the bounded retry included — is a PROMPT-LEVEL + * contract. No runtime code intercepts `transfer_task` failures, counts retry + * attempts, or recomputes the merged outcome; these tests pin the instruction + * text, not an enforcement mechanism. A model that violates the contract is + * caught by the fail-closed backstops instead: the aggregation rule keeps the + * merged result incomplete (never approving), and review-pr/action.yml's + * API-verified no-post detection (workflow-security tests) surfaces a run + * that finished without posting. + * * Like src/caller-permissions, this reads the YAML as text with a focused, * dependency-free extractor instead of pulling in a YAML parser. */ @@ -225,13 +236,57 @@ describe('semantic refusal rule (executable mirror)', () => { describe('root orchestration contracts', () => { const root = normalize(rootAgent); - it('treats a drafter transfer_task tool error like a malformed response, never retried', () => { + it('treats a drafter transfer_task tool error as the fallback object, never approving', () => { expect(root).toContain('A `transfer_task` tool error counts as exactly this same case'); expect(root).toContain( - 'do NOT retry the delegation and NEVER approve on the basis of an errored delegation', + "treat that delegation's response as the fallback object above and NEVER approve on the basis of an errored delegation", + ); + expect(root).toContain( + 'Do NOT retry it, with ONE narrow exception — the bounded runtime-delegation retry defined below', ); }); + it('bounds the drafter retry to recognized runtime delegation failures', () => { + // Prompt-level contract: the bound is instruction text the model follows, + // not runtime-enforced — nothing counts retries. These assertions pin the + // wording; the fail-closed backstops (aggregation rule, API-verified + // no-post detection) cover a model that ignores it. + expect(root).toContain( + 'Bounded retry for runtime delegation failures (REQUIRED — runs after the whole batch settles, BEFORE the aggregation rule below)', + ); + // Recognition is narrow: only the runtime rejecting the delegation itself + // (the batched-transfer misrouting seen in production) qualifies. + expect(root).toContain( + 'a delegation error is a RUNTIME DELEGATION FAILURE only when the `transfer_task` tool call itself failed', + ); + expect(root).toContain('"cannot transfer task"'); + expect(root).toContain('"target agent not in sub-agents list"'); + // Malformed/refusal output stays fail-closed and is never retried. + expect(root).toContain( + 'Drafter OUTPUT that is empty, malformed, refused, partial, or schema-rejected is NEVER a runtime delegation failure and is NEVER retried', + ); + // Sequential and bounded; successful chunks are retained. + expect(root).toContain( + 're-delegate each such chunk exactly ONCE, one at a time (sequential single `transfer_task` calls — never re-batch retries)', + ); + expect(root).toContain( + "keeping every successful delegation's response from the original batch", + ); + expect(root).toContain( + 'HARD BOUNDS: at most one retry per chunk, at most TWO retried chunks per review (when more than two chunks failed this way, retry the first two in chunk order and leave the rest as fallback objects), and one retry pass per review', + ); + // Posting reserve: retrying never eats the budget needed to post — an + // incomplete posted review always beats a timed-out silent one. + expect(root).toContain('SKIP the retry pass entirely when posting time is at risk'); + expect(root).toContain('a posted incomplete review beats a timeout'); + expect(root).toContain( + 'A retry never turns a failed or incomplete chunk into a success by itself — only a valid, complete JSON response does', + ); + // Retry attempts stay visible in the diagnostic merged summary. + expect(root).toContain('(retried after runtime delegation failure)'); + expect(root).toContain('(runtime delegation failure; sequential retry failed)'); + }); + it('routes malformed/refused drafter partial JSON through the incomplete-review fallback', () => { expect(root).toContain('partial, refused, or otherwise malformed'); expect(root).toContain( @@ -277,7 +332,7 @@ describe('root orchestration contracts', () => { "Merged `findings` = the concatenation of every delegation's `findings` array.", ); expect(root).toContain( - 'Merged `review_complete` = true ONLY IF every delegation returned valid JSON with `review_complete: true`.', + 'Merged `review_complete` = true ONLY IF every delegation returned valid JSON with `review_complete: true` (a chunk retried under the bounded runtime-delegation retry counts by its final result).', ); expect(root).toContain( 'If ANY delegation errored, was malformed, refused, or partial (i.e. became the fallback object above), or returned `review_complete: false`, the merged `review_complete` is false.', @@ -307,7 +362,7 @@ describe('root orchestration contracts', () => { 'If `review_complete` is `false` AND findings is non-empty → the review is INCOMPLETE, and it stays INCOMPLETE no matter what later steps find.', ); expect(root).toContain( - 'the posted review MUST use the incomplete-review body from Decision Rules rule 4 (the "### ⚠️ Review incomplete" heading plus the diagnostic merged `summary`) and MUST NOT carry a "🟢 APPROVE" label or any approve wording', + 'the posted review MUST use the incomplete-review body from Decision Rules rule 4 (the "### ⚠️ Review incomplete" heading plus the diagnostic merged `summary`) and MUST NOT carry a "🟢 NO FINDINGS" label or any approve wording', ); expect(root).toContain( 'Verified findings — however high their confidence — never overwrite incompleteness.', @@ -322,7 +377,7 @@ describe('root orchestration contracts', () => { expect(root).toContain( 'Do NOT emit an "### Assessment:" line and do NOT use approve wording anywhere in the review body.', ); - expect(root).toContain('a would-be "🟢 APPROVE" becomes "⚠️ INCOMPLETE" instead'); + expect(root).toContain('a would-be "🟢 NO FINDINGS" becomes "⚠️ INCOMPLETE" instead'); expect(root).toContain( 'Confidence scores decide per-finding dispositions only — they never restore a complete or approving outcome.', ); @@ -342,12 +397,12 @@ describe('root orchestration contracts', () => { ); }); - it('reserves the zero-findings 🟢 APPROVE template for complete merges', () => { + it('reserves the zero-findings 🟢 NO FINDINGS template for complete, conclusive, zero-surviving merges', () => { expect(root).toContain( - 'use only when findings are empty AND the merged `review_complete` is true', + 'use only when ZERO findings of ANY severity survive — none inline AND none in any summary list, low included — AND the merged `review_complete` is true AND verification, where required, was conclusive.', ); expect(root).toContain( - 'incomplete reviews must instead post the "### ⚠️ Review incomplete" body from Decision Rules rule 4, never a 🟢 APPROVE body', + 'Incomplete reviews must instead post the "### ⚠️ Review incomplete" body from Decision Rules rule 4, and inconclusive verification the "### ⚠️ Verification inconclusive" body — never a 🟢 NO FINDINGS body', ); }); @@ -374,7 +429,80 @@ describe('root orchestration contracts', () => { expect(root).toContain( 'or the `transfer_task` call itself fails or returns a tool error, which you MUST treat exactly like an empty/malformed response', ); - expect(root).toContain('A `transfer_task` tool error is never grounds to approve or to retry.'); + expect(root).toContain('A `transfer_task` tool error is never grounds to approve.'); + }); + + it('bounds the verifier retry to recognized runtime delegation failures', () => { + expect(root).toContain( + 'Do NOT retry the delegation for empty, malformed, refused, or partial verifier OUTPUT.', + ); + expect(root).toContain( + 'ONE bounded exception, mirroring step 5: a runtime delegation failure (the `transfer_task` call itself fails with e.g. "cannot transfer task" or "target agent not in sub-agents list") may be retried exactly once, sequentially', + ); + expect(root).toContain( + 'if the retry fails for any reason, apply this fallback — never a second retry', + ); + }); + + it('keeps the inconclusive-verification fallback off the completion marker', () => { + expect(root).toContain( + 'The inconclusive-verification body MUST open with "### ⚠️ Verification inconclusive"', + ); + expect(root).toContain('never an "### Assessment:" line and never approve wording'); + expect(root).toContain('an unverified review must never carry it'); + }); + + it('surfaces surviving low findings instead of dropping them', () => { + // The #1814 regression: low findings skipped verification AND silently + // vanished from the review, which then claimed a clean result. + expect(root).toContain( + 'Skip verification for "low" findings — but NEVER drop them: every surviving "low" finding stays in the review as a summary-only entry', + ); + expect(root).toContain('Low-severity findings (not verified, not posted inline)'); + expect(root).toContain( + 'blocks the 🟢 NO FINDINGS label like any other surviving finding (Decision Rules rules 1–3)', + ); + // Step 9 builds the dedicated review-body section. + expect(root).toContain('**Low-severity summary** — surviving low-severity findings'); + expect(root).toContain( + 'Never silently drop these: each one blocks the 🟢 NO FINDINGS label (Decision Rules rule 3).', + ); + }); + + it('drives the assessment from every surviving finding, approving only on zero', () => { + // The TS module is an executable spec/mirror of these rules, honestly + // labeled as such — the prompt must never claim a runtime enforcer exists. + expect(root).toContain( + 'YOU apply these rules — nothing else recomputes the assessment at runtime.', + ); + expect(root).toContain( + '`src/review-assessment/review-assessment.ts` is their executable spec: an outcome-for-outcome mirror pinned by unit tests, not a runtime enforcer.', + ); + expect(root).not.toContain('authoritative implementation of rules 1–4'); + expect(root).toContain('**Collect the SURVIVING findings**'); + expect(root).toContain( + 'inline comments, the lower-confidence summary, the medium-severity floor list, and the unverified low-severity list', + ); + expect(root).toContain( + 'ANY other surviving finding (NOTABLE or MINOR — any severity, any surfaced disposition) → label as "🟡 NEEDS ATTENTION"', + ); + expect(root).toContain('EXACTLY ZERO surviving findings → label as "🟢 NO FINDINGS"'); + expect(root).toContain( + '"🟢 NO FINDINGS" is emitted ONLY for a complete review (merged `review_complete` true) with conclusive verification and zero surviving findings of EVERY severity.', + ); + expect(root).toContain( + 'A review that surfaces ANY finding — inline or summary-only, including a single unverified low — must NOT carry the NO FINDINGS label.', + ); + expect(root).toContain( + 'The label is a neutral completion marker, NEVER an approval: the bot never approves a PR, so no body may say "APPROVE", "LGTM", or "No issues found".', + ); + }); + + it('lists low-severity findings in the console format', () => { + expect(root).toContain('### Low-severity findings (not verified, not posted inline)'); + expect(root).toContain( + 'Omit the "Lower-confidence", "Low-severity", and "Dismissed security" sections when they have no entries.', + ); }); it('assigns deterministic finding_ids before delegating to the verifier', () => { @@ -437,7 +565,6 @@ describe('root orchestration contracts', () => { describe('verifier instruction contracts', () => { const verifier = normalize(verifierAgent); - it('demands exactly one verdict per finding, paired by finding_id', () => { expect(verifier).toContain('You MUST produce exactly one verdict per finding'); expect(verifier).toContain('never merge, split, invent, or pad verdicts'); @@ -464,3 +591,101 @@ describe('verifier instruction contracts', () => { expect(verifier).not.toContain('the schema rejects it'); }); }); + +describe('posting template hazards', () => { + const template = readFileSync( + resolve(import.meta.dirname, '../../../review-pr/agents/refs/posting-format.md'), + 'utf-8', + ); + + it('has no REVIEW_BODY assignment at all — posting is guarded on the computed outcome', () => { + // The old template pre-assigned an assessment badge to a REVIEW_BODY shell + // variable; every run that copied it verbatim posted a label it never + // computed (template bleed). Now no shell variable exists at all: the body + // is a quoted-heredoc file, and the chained posting command reaches + // `gh api` only after `test -s` and the trusted finalize-body validator + // accept it (exactly one computed status line, staged comments file parsed + // and 🟢 NO FINDINGS refused over any staged comment, marker appended + // mechanically). The workflow-security harness executes this chain. + expect(template).not.toMatch(/^REVIEW_BODY=/m); + expect(template).not.toMatch(/\$REVIEW_BODY|\$\{REVIEW_BODY/); + expect(template).toContain("cat > /tmp/review_body.md << 'REVIEW_BODY_EOF'"); + expect(template).toContain('test -s /tmp/review_body.md \\'); + expect(template).toContain( + '&& node /tmp/review-assessment.js finalize-body /tmp/review_body.md __REVIEW_RUN_NONCE__ /tmp/review_comments.json \\', + ); + expect(template).toContain('--rawfile body /tmp/review_body.md'); + expect(template).toContain('(the ONLY zero-findings outcome'); + // The payload is staged to a trusted temp file and validated so `gh api` + // is never invoked when jq fails to construct it (no `jq | gh` pipe). + expect(template).toContain('> /tmp/review_payload.json \\'); + expect(template).toContain( + `&& jq -e 'type == "object"' /tmp/review_payload.json > /dev/null \\`, + ); + expect(template).not.toMatch(/\|\s*gh api/); + }); + + it('routes through action-staged placeholders, not literal {owner}/{repo}/{pr}', () => { + // Every recent successful run first 404ed on the literal route before the + // model hand-corrected it; trusted routing data is staged by the action. + expect(template).not.toMatch(/\{owner\}|\{repo\}|\{pr\}/); + expect(template).toContain( + '&& gh api "repos/__REPOSITORY__/pulls/__PR_NUMBER__/reviews" --input - < /tmp/review_payload.json', + ); + expect(template).toContain('--arg commit_id "__PR_HEAD_SHA__"'); + expect(template).toContain('Run the chained command exactly as rendered'); + }); + + it('documents the non-approving body outcomes, including the low-severity list', () => { + expect(template).toContain('"### ⚠️ Review incomplete" (never an "### Assessment:" line)'); + expect(template).toContain('"### ⚠️ Verification inconclusive"'); + expect(template).toContain('#### Low-severity findings (not verified, not posted inline)'); + expect(template).toContain( + 'complete review, conclusive verification, ZERO surviving findings of every', + ); + }); +}); + +describe('COMMENT-event and neutral zero-findings label invariants', () => { + const template = readFileSync( + resolve(import.meta.dirname, '../../../review-pr/agents/refs/posting-format.md'), + 'utf-8', + ); + + // Matches an event being SET to an approving value (jq --arg, JSON, YAML, or + // shell assignment) while skipping prose prohibitions like "never `APPROVE`", + // where words separate "event" from the value. + const approvingEventAssignment = /event\W{0,4}(?:APPROVE|REQUEST_CHANGES)/; + + it('pins the COMMENT event and prohibits the approving events in the instructions', () => { + const root = normalize(rootAgent); + expect(root).toContain( + 'ALWAYS use the `COMMENT` event — never `APPROVE` or `REQUEST_CHANGES`.', + ); + expect(root).toContain( + 'The GitHub review event is ALWAYS `COMMENT`, regardless of the assessment label. Never use `APPROVE` or `REQUEST_CHANGES`.', + ); + }); + + it('never sets an APPROVE or REQUEST_CHANGES event in the yaml or the posting template', () => { + expect(source).not.toMatch(approvingEventAssignment); + expect(template).not.toMatch(approvingEventAssignment); + // The posting command hardcodes the COMMENT event, and no other event + // value is ever passed to jq. + expect(template).toContain('--arg event "COMMENT"'); + expect(template.match(/--arg event "/g)).toEqual(['--arg event "']); + }); + + it('keeps the retired 🟢 APPROVE label and legacy LGTM wording out of the active policy', () => { + // The zero-findings outcome is the neutral 🟢 NO FINDINGS label; only + // historical reviews may carry "### Assessment: 🟢 APPROVE", and only + // src/incremental-review's marker matching still recognizes them. + for (const text of [source, template]) { + expect(text).not.toContain('🟢 APPROVE'); + expect(text).not.toContain('LGTM!'); + expect(text).not.toContain('🟢 **No issues found**'); + } + expect(source).toContain('label as "🟢 NO FINDINGS"'); + expect(template).toContain('"### Assessment: 🟢 NO FINDINGS"'); + }); +}); diff --git a/src/rate-limit/__tests__/rate-limit.test.ts b/src/rate-limit/__tests__/rate-limit.test.ts index 320de36..16af34b 100644 --- a/src/rate-limit/__tests__/rate-limit.test.ts +++ b/src/rate-limit/__tests__/rate-limit.test.ts @@ -56,7 +56,7 @@ function reply(secAgo: number, marker = REPLY_MARKER, login = BOT) { // A full review posted via the Reviews API: one per review LLM run, identified by // bot author plus a non-empty assessment/status body (no inline marker). -function review(secAgo: number, body = '### Assessment: 🟢 APPROVE', login = BOT) { +function review(secAgo: number, body = '### Assessment: 🟢 NO FINDINGS', login = BOT) { return { user: { login }, body, submitted_at: within(secAgo) }; } @@ -100,30 +100,77 @@ describe('detectRateAnomaly', () => { expect(r.threshold).toBe(3); }); - it('counts a zero-finding APPROVE review (no marker, non-empty body)', async () => { + it('counts zero-finding completion reviews (no marker, non-empty body)', async () => { // Regression: review bodies have no inline marker and zero-finding reviews - // post no inline comments, so this is invisible to the comment endpoints. - routePaginate([], [], [review(30, '### Assessment: 🟢 APPROVE')]); - const r = await detectRateAnomaly('tok', { ...base, threshold: 1 }); - expect(r.count).toBe(1); + // post no inline comments, so these are invisible to the comment endpoints. + routePaginate( + [], + [], + [ + review(30, '### Assessment: 🟢 NO FINDINGS'), + // Legacy label posted before the zero-findings label was neutralized — + // still one review run. + review(60, '### Assessment: 🟢 APPROVE'), + ], + ); + const r = await detectRateAnomaly('tok', { ...base, threshold: 2 }); + expect(r.count).toBe(2); expect(r.anomalous).toBe(true); }); - it('counts timeout / error / LGTM fallback reviews (no marker)', async () => { + it('counts timeout / error / incomplete fallback reviews (no marker)', async () => { routePaginate( [], [], [ review(30, '⏱️ **PR Review Timed Out** — …'), review(60, '❌ **PR Review Failed** — …'), - review(90, '🟢 **No issues found** — LGTM!'), + review(90, '⚠️ **Review incomplete** — The review agent finished without posting a review.'), + // Legacy no-post fallback bodies still exist on old PRs and still count. + review(120, '🟢 **No issues found** — LGTM!'), ], ); const r = await detectRateAnomaly('tok', base); - expect(r.count).toBe(3); + expect(r.count).toBe(4); expect(r.anomalous).toBe(true); }); + it('counts run-marker-bearing reviews posted through other [bot] identities', async () => { + // The action's github-token input defaults to github.token, which posts + // as github-actions[bot] — those runs are recognized by the per-run + // attribution marker their review bodies (fallback notices included) carry. + const marker = ``; + routePaginate( + [], + [], + [ + review(30, `### Assessment: 🟡 NEEDS ATTENTION\n\n${marker}`, 'github-actions[bot]'), + review(60, `⏱️ **PR Review Timed Out** — …\n\n${marker}`, 'github-actions[bot]'), + ], + ); + const r = await detectRateAnomaly('tok', { ...base, threshold: 2 }); + expect(r.count).toBe(2); + expect(r.anomalous).toBe(true); + }); + + it('never counts marker-bearing reviews from non-[bot] logins or unmarked bot reviews', async () => { + // The marker format is public: a human pasting it into a review must not + // inflate the count (griefing the throttle), and an unmarked review from + // an unrelated [bot] integration is not an action output. + const marker = ``; + routePaginate( + [], + [], + [ + review(30, `### Assessment: 🟢 NO FINDINGS\n\n${marker}`, 'mallory'), + review(60, '### Assessment: 🟢 NO FINDINGS', 'github-actions[bot]'), + ], + ); + const r = await detectRateAnomaly('tok', { ...base, threshold: 1 }); + expect(r.count).toBe(0); + expect(r.anomalous).toBe(false); + }); + it('is not anomalous below the threshold', async () => { routePaginate([reply(120)], [], [review(30)]); const r = await detectRateAnomaly('tok', base); @@ -162,7 +209,7 @@ describe('detectRateAnomaly', () => { routePaginate( [{ user: { login: 'mallory' }, body: `spam ${REPLY_MARKER}`, created_at: within(10) }], [], - [review(20, '### Assessment: 🟢 APPROVE', 'mallory')], + [review(20, '### Assessment: 🟢 NO FINDINGS', 'mallory')], ); const r = await detectRateAnomaly('tok', base); expect(r.count).toBe(0); @@ -173,7 +220,7 @@ describe('detectRateAnomaly', () => { routePaginate( [reply(60, REPLY_MARKER, 'docker-agent[bot]')], [], - [review(30, '### Assessment: 🟢 APPROVE', 'docker-agent[bot]')], + [review(30, '### Assessment: 🟢 NO FINDINGS', 'docker-agent[bot]')], ); const r = await detectRateAnomaly('tok', { ...base, threshold: 2 }); expect(r.count).toBe(2); diff --git a/src/rate-limit/index.ts b/src/rate-limit/index.ts index 9b04153..5ede17c 100644 --- a/src/rate-limit/index.ts +++ b/src/rate-limit/index.ts @@ -15,8 +15,9 @@ * * Counting is per LLM run, so each run contributes exactly one unit: * - Reviews are posted via the Reviews API (POST /pulls/{n}/reviews) with no - * inline marker — a findings review, a zero-finding APPROVE, and the - * timeout/error/LGTM fallbacks all land there. They are counted from + * inline marker — a findings review, a zero-finding 🟢 NO FINDINGS + * completion, and the timeout/error/incomplete fallbacks all land there. + * They are counted from * `pulls.listReviews` by bot author (a real review run always carries an * assessment/status body); the inline finding comments such a review carries * are deliberately not counted, since that would be N units per single run. @@ -40,6 +41,7 @@ */ import * as core from '@actions/core'; import { Octokit } from '@octokit/rest'; +import { isActionPostedReview, matchesBotLogin } from '../review-assessment/review-assessment.js'; // Reply markers identify the bot's conversational replies — one per reply LLM // run — posted as issue comments or inline review-comment replies. Full reviews @@ -50,12 +52,11 @@ import { Octokit } from '@octokit/rest'; // countable during migration. const REPLY_MARKERS = ['', '']; -// GitHub presents the bot identity as "docker-agent" when posting with a machine -// user token, or "docker-agent[bot]" through a GitHub App installation token. -// Match both so the count is correct regardless of which token posted. -function matchesBotLogin(login: string | null | undefined, botLogin: string): boolean { - return login === botLogin || login === `${botLogin}[bot]`; -} +// The bot posts as "docker-agent" (machine user token) or "docker-agent[bot]" +// (GitHub App installation token); reviews posted through the action's public +// github-token input under another `[bot]` identity are recognized by the +// per-run attribution marker their bodies carry (isActionPostedReview — shared +// with src/incremental-review). export interface RateAnomalyOptions { owner: string; @@ -100,9 +101,10 @@ function isAgentReplyComment(c: CommentLike, botLogin: string, windowStartMs: nu } function isAgentReview(r: ReviewLike, botLogin: string, windowStartMs: number): boolean { - if (!matchesBotLogin(r.user?.login, botLogin)) return false; + if (!isActionPostedReview(r.user?.login, r.body, botLogin)) return false; // A real review run always carries an assessment/status body ("### Assessment: - // …", or a timeout/error/LGTM fallback). Standalone inline comments and replies + // …", or a timeout/error/incomplete fallback). Standalone inline comments and + // replies // surface in this endpoint as empty-body review entries; skipping them keeps // each review run counted exactly once and avoids double-counting an inline // reply (already counted via its reply marker on the comment endpoints). diff --git a/src/resolve-trigger-context/__tests__/workflow-security.test.ts b/src/resolve-trigger-context/__tests__/workflow-security.test.ts index cefbbab..2fb0ee4 100644 --- a/src/resolve-trigger-context/__tests__/workflow-security.test.ts +++ b/src/resolve-trigger-context/__tests__/workflow-security.test.ts @@ -6,20 +6,33 @@ import { chmodSync, cpSync, existsSync, + mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { delimiter, resolve } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { delimiter, dirname, resolve } from 'node:path'; +import { beforeAll, describe, expect, it } from 'vitest'; import { parseDocument } from 'yaml'; +import { findLastReviewedSha } from '../../incremental-review/incremental-review.js'; +import { builtReviewAssessmentCli } from '../../review-assessment/__tests__/build-cli.js'; import { resolverOutputs } from '../index.js'; import type { CanonicalComment, CanonicalTriggerContext } from '../resolve-trigger-context.js'; const root = resolve(import.meta.dirname, '../../..'); -const safePath = '/usr/bin:/bin'; +// The review-assessment CLI harnesses run `node` from the rendered scripts, +// so the spawning node's own directory is prepended to the deterministic +// system-tool path. +const safePath = `${dirname(process.execPath)}${delimiter}/usr/bin:/bin`; + +// Bundled once per test process (tsup, same shape as `pnpm build`): the +// production scripts under test execute this exact runtime artifact. +let reviewAssessmentCli = ''; +beforeAll(async () => { + reviewAssessmentCli = await builtReviewAssessmentCli(); +}); function testEnvironment(values: Record): NodeJS.ProcessEnv { return { @@ -578,20 +591,39 @@ function requiredComment(context: CanonicalTriggerContext): CommentContext { return context.comment; } -function actionStepRun(name: string): string { +function reviewActionSteps(): WorkflowStep[] { const action = parseDocument( readFileSync(resolve(root, 'review-pr/action.yml'), 'utf8'), ).toJS() as Action; - const matches = action.runs?.steps?.filter((candidate) => candidate.name === name) ?? []; - if (matches.length !== 1 || !matches[0].run) throw new Error(`Expected one ${name} run body`); - return matches[0].run; + return action.runs?.steps ?? []; +} + +function reviewActionStep(name: string): WorkflowStep { + const matches = reviewActionSteps().filter((candidate) => candidate.name === name); + if (matches.length !== 1) throw new Error(`Expected one ${name} step`); + return matches[0]; +} + +function actionStepRun(name: string): string { + const run = reviewActionStep(name).run; + if (!run) throw new Error(`Expected a ${name} run body`); + return run; } function summaryRun(): string { return actionStepRun('Post clean summary'); } -function runCopyReference(headSha: string, template: string): ReturnType { +const TEST_NONCE = '0123456789abcdef0123456789abcdef'; +const TEST_MARKER = ``; + +function runCopyReference( + headSha: string, + template: string, + repository = 'docker/docker-agent-action', + prNumber = '88', + runNonce = TEST_NONCE, +): { result: ReturnType; rendered: string } { const directory = mkdtempSync(resolve(tmpdir(), 'docker-agent-copy-reference-')); const actionPath = resolve(directory, 'action'); const refs = resolve(actionPath, 'agents/refs'); @@ -606,6 +638,9 @@ function runCopyReference(headSha: string, template: string): ReturnType> "$GH_RECORDS" +if [[ "$args" == "${summaryReviewsRoute}" ]]; then + if [[ "\${GH_REVIEWS_FETCH_FAILS:-}" == "1" ]]; then exit 1; fi + cat "$GH_REVIEWS_FILE" +fi +if [[ "$args" == *" --input -" ]]; then + if [[ -z "$input" ]]; then exit 22; fi + if [[ "\${GH_POST_FAILS:-}" == "1" ]]; then exit 1; fi +fi +`; +} + function runSummary(invocation: SummaryInvocation): { result: ReturnType; records: GhRecord[]; summary: string; + outputs: string; directory: string; } { const directory = mkdtempSync(resolve(tmpdir(), 'docker-agent-summary-')); const output = resolve(directory, 'output'); const summary = resolve(directory, 'summary'); const recordsPath = resolve(directory, 'gh-records.jsonl'); + const reviewsPath = resolve(directory, 'reviews.json'); writeFileSync(recordsPath, ''); writeFileSync(output, ''); writeFileSync(summary, ''); + writeFileSync(reviewsPath, JSON.stringify(invocation.reviews ?? [])); const reference = invocation.postingReference ?? resolve(directory, 'posting-format.md'); const sha = invocation.headSha ?? 'a'.repeat(40); + const nonce = invocation.nonce ?? TEST_NONCE; writeFileSync(resolve(directory, 'summary.sh'), summaryRun()); if (!invocation.postingReference) { - writeFileSync(reference, `jq -n --arg commit_id "${sha}" '{commit_id: $commit_id}'`); + writeFileSync( + reference, + [ + `node /tmp/review-assessment.js finalize-body /tmp/review_body.md ${TEST_NONCE} /tmp/review_comments.json`, + `jq -n --arg commit_id "${sha}" '{commit_id: $commit_id}' | gh api "repos/docker/docker-agent-action/pulls/88/reviews" --input -`, + ].join('\n'), + ); } + // The summary step classifies via the bundled CLI at $ACTION_PATH/../dist. + const actionPath = resolve(directory, 'action'); + mkdirSync(actionPath, { recursive: true }); + mkdirSync(resolve(directory, 'dist'), { recursive: true }); + cpSync(reviewAssessmentCli, resolve(directory, 'dist/review-assessment.js')); if (invocation.verboseLog !== undefined) writeFileSync(resolve(directory, 'verbose.log'), invocation.verboseLog); - writeFileSync( - resolve(directory, 'gh'), - `#!/usr/bin/env bash -set -euo pipefail -args="$*" -input="" -if [[ "$args" == *" --input -" ]]; then input=$(cat); fi -printf '%s\\n' "$(jq -cn --arg args "$args" --arg input "$input" '{args: $args, input: $input}')" >> "$GH_RECORDS" -if [[ "$args" == *"/reviews --jq "* ]]; then - if [[ "$args" == *"/issues/"* ]]; then printf '%s\\n' "${invocation.dedupCounts?.[1] ?? 0}"; else printf '%s\\n' "${invocation.dedupCounts?.[0] ?? 0}"; fi -fi -`, - ); + writeFileSync(resolve(directory, 'gh'), ghMock()); chmodSync(resolve(directory, 'gh'), 0o755); const result = spawnSync( '/bin/bash', @@ -696,6 +782,9 @@ fi env: testEnvironment({ PATH: `${directory}${delimiter}${safePath}`, GH_RECORDS: recordsPath, + GH_REVIEWS_FILE: reviewsPath, + GH_REVIEWS_FETCH_FAILS: invocation.reviewsFetchFails ? '1' : '', + GH_POST_FAILS: invocation.postFails ? '1' : '', GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: summary, REPOSITORY: 'docker/docker-agent-action', @@ -707,9 +796,11 @@ fi invocation.verboseLog === undefined ? '' : resolve(directory, 'verbose.log'), CHUNK_COUNT: invocation.chunkCount ?? '', LOCK_AGE: '', - ACTION_PATH: directory, + ACTION_PATH: actionPath, PR_HEAD_SHA: sha, POSTING_REFERENCE: reference, + BASELINE_MAX_REVIEW_ID: invocation.baseline ?? '100', + RUN_NONCE: nonce, }), encoding: 'utf8', }, @@ -719,7 +810,13 @@ fi .split('\n') .filter(Boolean) .map((line) => JSON.parse(line) as GhRecord); - return { result, records, summary: readFileSync(summary, 'utf8'), directory }; + return { + result, + records, + summary: readFileSync(summary, 'utf8'), + outputs: readFileSync(output, 'utf8'), + directory, + }; } function reviewCreations(records: GhRecord[]): GhRecord[] { @@ -1148,28 +1245,174 @@ describe('fork workflow security regressions', () => { expect(kills).toBe(72); }); + const summarySha = 'a'.repeat(40); + const summaryOtherSha = 'b'.repeat(40); + const postedReview = (id = 101, commitId = summarySha, login = 'docker-agent'): ReviewState => ({ + id, + user: { login }, + commit_id: commitId, + body: `### Assessment: 🟡 NEEDS ATTENTION\n\n${TEST_MARKER}\n`, + state: 'COMMENTED', + }); + const unmarkedReview = ( + id = 101, + commitId = summarySha, + login = 'docker-agent', + ): ReviewState => ({ + id, + user: { login }, + commit_id: commitId, + body: '### Assessment: 🟡 NEEDS ATTENTION', + state: 'COMMENTED', + }); + const agentStatusReview = (header: string): ReviewState => ({ + id: 101, + user: { login: 'github-actions[bot]' }, + commit_id: summarySha, + body: `${header}\nchunk 2: Drafter did not complete\n\n${TEST_MARKER}\n`, + state: 'COMMENTED', + }); + const noticeReview = (commitId: string, login = 'docker-agent'): ReviewState => ({ + id: 60, + user: { login }, + commit_id: commitId, + body: '⚠️ **Review incomplete** — The review agent finished without posting a review.', + state: 'COMMENTED', + }); + it.each([ { - name: 'normal agent-posted success', + name: 'API-verified success (verbose log says nothing)', exitCode: '0', - verboseLog: 'pullrequestreview-1', - reads: 0, + verboseLog: 'no review', + reviews: [postedReview()], + status: 'completed', + body: undefined, + }, + { + name: 'exit 0 with only a log-quoted review ID (no-post fallback)', + // The regression the API baseline fixes: an old review ID mentioned in + // the verbose log must never count as evidence this run posted a review. + exitCode: '0', + verboseLog: 'replying about pullrequestreview-999 from an old run', + reviews: [], + status: 'incomplete', + body: '⚠️ \\*\\*Review incomplete\\*\\*', + }, + { + name: 'exit 0 with only a fresh human review on a different SHA', + exitCode: '0', + verboseLog: 'no review', + reviews: [unmarkedReview(101, summaryOtherSha, 'human-reviewer')], + status: 'incomplete', + body: '⚠️ \\*\\*Review incomplete\\*\\*', + }, + { + name: 'exit 0 with only a pre-baseline unmarked bot review on the selected SHA', + exitCode: '0', + verboseLog: 'no review', + reviews: [unmarkedReview(100)], + status: 'incomplete', + body: '⚠️ \\*\\*Review incomplete\\*\\*', + }, + { + name: 'exit 0 without a verbose log and no posted review', + exitCode: '0', + reviews: [], + status: 'incomplete', + body: '⚠️ \\*\\*Review incomplete\\*\\*', + }, + { + name: 'exit 0 with a fresh marker review posted by the app-token identity', + exitCode: '0', + verboseLog: 'no review', + reviews: [postedReview(101, summarySha, 'docker-agent[bot]')], + status: 'completed', + body: undefined, + }, + { + // The github-token input defaults to github.token, which posts as + // github-actions[bot]; a custom PAT posts as an arbitrary machine user. + // Attribution is by exact marker, so both complete. + name: 'exit 0 with a fresh marker review posted by the default-token identity', + exitCode: '0', + verboseLog: 'no review', + reviews: [postedReview(101, summarySha, 'github-actions[bot]')], + status: 'completed', body: undefined, }, { - name: 'zero-findings already posted', + name: 'exit 0 with a fresh marker review posted by a consumer machine user', exitCode: '0', verboseLog: 'no review', - dedupCounts: [1, 0], - reads: 2, + reviews: [postedReview(101, summarySha, 'consumer-machine-user')], + status: 'completed', body: undefined, }, + { + name: 'exit 0 with a fresh same-SHA unmarked review from a human (unrelated)', + exitCode: '0', + verboseLog: 'no review', + reviews: [unmarkedReview(101, summarySha, 'human-reviewer')], + status: 'incomplete', + body: '⚠️ \\*\\*Review incomplete\\*\\*', + }, + { + // The agent honestly posted an incomplete review (unreviewed chunks): + // that semantic status stands — no duplicate notice, never completed. + name: 'exit 0 with an agent-posted incomplete review body', + exitCode: '0', + verboseLog: 'no review', + reviews: [agentStatusReview('### ⚠️ Review incomplete')], + status: 'incomplete', + body: undefined, + }, + { + name: 'exit 0 with an agent-posted inconclusive review body', + exitCode: '0', + verboseLog: 'no review', + reviews: [agentStatusReview('### ⚠️ Verification inconclusive')], + status: 'inconclusive', + body: undefined, + }, + { + name: 'incomplete notice already posted for this SHA', + exitCode: '0', + verboseLog: 'no review', + reviews: [noticeReview(summarySha)], + status: 'incomplete', + body: undefined, + }, + { + name: 'incomplete notice from the app-token identity dedups', + exitCode: '0', + verboseLog: 'no review', + reviews: [noticeReview(summarySha, 'docker-agent[bot]')], + status: 'incomplete', + body: undefined, + }, + { + name: 'incomplete notice from a human on this SHA never dedups', + exitCode: '0', + verboseLog: 'no review', + reviews: [noticeReview(summarySha, 'human-reviewer')], + status: 'incomplete', + body: '⚠️ \\*\\*Review incomplete\\*\\*', + }, + { + name: 'incomplete notice on another SHA never dedups', + exitCode: '0', + verboseLog: 'no review', + reviews: [noticeReview(summaryOtherSha)], + status: 'incomplete', + body: '⚠️ \\*\\*Review incomplete\\*\\*', + }, { name: 'timeout with unknown chunks', exitCode: '124', verboseLog: 'no review', chunkCount: '', - reads: 0, + status: 'timed-out', body: '⏱️', }, { @@ -1177,7 +1420,7 @@ describe('fork workflow security regressions', () => { exitCode: '124', verboseLog: 'no review', chunkCount: '1', - reads: 0, + status: 'timed-out', body: '⏱️', }, { @@ -1185,37 +1428,364 @@ describe('fork workflow security regressions', () => { exitCode: '124', verboseLog: 'no review', chunkCount: '2', - reads: 0, + status: 'timed-out', + body: '⏱️', + }, + { + name: 'timeout after this run posted its review (no redundant fallback)', + exitCode: '124', + verboseLog: 'no review', + reviews: [postedReview()], + status: 'completed-with-warnings', + body: undefined, + }, + { + // Exit 124 after an agent-posted incomplete review keeps the semantic + // status — no duplicate timeout fallback next to the posted review. + name: 'timeout after an agent-posted incomplete review', + exitCode: '124', + verboseLog: 'no review', + chunkCount: '1', + reviews: [agentStatusReview('### ⚠️ Review incomplete')], + status: 'incomplete', + body: undefined, + }, + { + name: 'timeout after an agent-posted inconclusive review', + exitCode: '124', + verboseLog: 'no review', + chunkCount: '1', + reviews: [agentStatusReview('### ⚠️ Verification inconclusive')], + status: 'inconclusive', + body: undefined, + }, + { + name: 'timeout with only a fresh same-SHA human review still posts the fallback', + exitCode: '124', + verboseLog: 'no review', + chunkCount: '1', + reviews: [unmarkedReview(101, summarySha, 'human-reviewer')], + status: 'timed-out', body: '⏱️', }, - { name: 'non-124 failure', exitCode: '1', verboseLog: 'no review', reads: 0, body: '❌' }, { - name: 'failure with prior review', + name: 'non-124 failure without a posted review despite a log marker', exitCode: '1', verboseLog: 'pullrequestreview-1', - reads: 0, + reviews: [], + status: 'failed', + body: '❌', + }, + { + name: 'non-124 failure with only a fresh same-SHA human review', + exitCode: '1', + verboseLog: 'no review', + reviews: [unmarkedReview(101, summarySha, 'human-reviewer')], + status: 'failed', + body: '❌', + }, + { + name: 'non-124 failure with an API-verified posted review', + exitCode: '1', + verboseLog: 'no review', + reviews: [postedReview()], + status: 'completed-with-warnings', body: undefined, }, { - name: 'fallback LGTM', - exitCode: '0', + // Nonzero exit after an agent-posted incomplete review keeps the + // semantic status — no duplicate failure fallback. + name: 'non-124 failure after an agent-posted incomplete review', + exitCode: '1', verboseLog: 'no review', - dedupCounts: [0, 0], - reads: 2, - body: '🟢', + reviews: [agentStatusReview('### ⚠️ Review incomplete')], + status: 'incomplete', + body: undefined, + }, + { + name: 'non-124 failure after an agent-posted inconclusive review', + exitCode: '1', + verboseLog: 'no review', + reviews: [agentStatusReview('### ⚠️ Verification inconclusive')], + status: 'inconclusive', + body: undefined, }, - { name: 'success without log', exitCode: '0', reads: 0, body: undefined }, ])('executes the summary $name vector with exact review payload behavior', (vector) => { - const sha = 'a'.repeat(40); const run = runSummary(vector); try { expect(run.result.status, run.result.stderr).toBe(0); + expect(run.outputs).toContain(`review-status=${vector.status}`); + // Exactly one authoritative API state lookup; the old per-branch --jq + // queries are gone. + expect(run.records.filter((record) => record.args === summaryReviewsRoute)).toHaveLength(1); + expect(run.records.filter((record) => record.args.includes(' --jq '))).toHaveLength(0); const creations = reviewCreations(run.records); - expect(run.records.filter((record) => record.args.includes(' --jq '))).toHaveLength( - vector.reads, - ); expect(creations).toHaveLength(vector.body ? 1 : 0); - if (vector.body) expectReviewPayload(creations[0], sha, vector.body); + if (vector.body) expectReviewPayload(creations[0], summarySha, vector.body); + // Fail-closed: no summary-step fallback may ever synthesize an approval + // or advance the incremental checkpoint (the false-LGTM regression: + // docker/gordon PRs #1798/#1803/#1808/#1809). + for (const creation of creations) { + const body = (JSON.parse(creation.input) as { body: string }).body; + expect(body).not.toContain('### Assessment:'); + expect(body).not.toMatch(/LGTM|No issues found/); + // The trusted step mechanically embeds this run's attribution marker + // in every fallback notice — rate counting stays login-independent. + expect(body).toContain(TEST_MARKER); + // Never a checkpoint, whichever identity posted the fallback. + for (const login of ['docker-agent', 'github-actions[bot]']) { + expect( + findLastReviewedSha([ + { + user: { login }, + body, + commit_id: summarySha, + submitted_at: '2026-01-01T00:00:00Z', + }, + ]), + ).toBeNull(); + } + } + } finally { + rmSync(run.directory, { recursive: true, force: true }); + } + }); + + it.each([ + { + name: 'missing pre-run baseline', + invocation: { exitCode: '0', verboseLog: 'no review', baseline: '' }, + diagnostic: 'Pre-run review baseline is missing or unreadable', + apiCalls: 0, + }, + { + name: 'garbled pre-run baseline', + invocation: { exitCode: '0', verboseLog: 'no review', baseline: '12x' }, + diagnostic: 'Pre-run review baseline is missing or unreadable', + apiCalls: 0, + }, + { + name: 'missing run attribution nonce', + invocation: { exitCode: '0', verboseLog: 'no review', nonce: '' }, + diagnostic: 'Run attribution nonce is missing or malformed', + apiCalls: 0, + }, + { + name: 'malformed run attribution nonce', + invocation: { exitCode: '0', verboseLog: 'no review', nonce: 'abc123' }, + diagnostic: 'Run attribution nonce is missing or malformed', + apiCalls: 0, + }, + { + name: 'post-run review lookup failure on exit 0', + invocation: { exitCode: '0', verboseLog: 'no review', reviewsFetchFails: true }, + diagnostic: 'Post-run review lookup failed', + apiCalls: 1, + }, + { + name: 'post-run review lookup failure on a nonzero exit', + invocation: { exitCode: '1', verboseLog: 'pullrequestreview-1', reviewsFetchFails: true }, + diagnostic: 'Post-run review lookup failed', + apiCalls: 1, + }, + { + // A stale/copied marker off this run's SHA or baseline is exactly the + // ambiguity the classifier refuses to interpret. + name: 'marker-bearing review on a different SHA', + invocation: { + exitCode: '0', + verboseLog: 'no review', + reviews: [postedReview(101, summaryOtherSha)], + }, + diagnostic: 'Posted-review state is ambiguous', + apiCalls: 1, + }, + { + name: 'marker-bearing review at the pre-run baseline', + invocation: { exitCode: '0', verboseLog: 'no review', reviews: [postedReview(100)] }, + diagnostic: 'Posted-review state is ambiguous', + apiCalls: 1, + }, + { + name: 'duplicate exact-marker reviews', + invocation: { + exitCode: '0', + verboseLog: 'no review', + reviews: [postedReview(101), postedReview(102)], + }, + diagnostic: 'Posted-review state is ambiguous', + apiCalls: 1, + }, + { + name: 'marker-bearing review in a non-COMMENTED state', + invocation: { + exitCode: '0', + verboseLog: 'no review', + reviews: [{ ...postedReview(), state: 'PENDING' }], + }, + diagnostic: 'Posted-review state is ambiguous', + apiCalls: 1, + }, + { + name: 'marker-bearing review in an APPROVED state', + invocation: { + exitCode: '0', + verboseLog: 'no review', + reviews: [{ ...postedReview(), state: 'APPROVED' }], + }, + diagnostic: 'Posted-review state is ambiguous', + apiCalls: 1, + }, + { + name: 'marker-bearing review without a status line (malformed body)', + invocation: { + exitCode: '0', + verboseLog: 'no review', + reviews: [{ ...postedReview(), body: `some prose\n\n${TEST_MARKER}\n` }], + }, + diagnostic: 'Posted-review state is ambiguous', + apiCalls: 1, + }, + { + name: 'marker-bearing review with conflicting status markers', + invocation: { + exitCode: '0', + verboseLog: 'no review', + reviews: [ + { + ...postedReview(), + body: `### ⚠️ Review incomplete\n### Assessment: 🟢 NO FINDINGS\n\n${TEST_MARKER}\n`, + }, + ], + }, + diagnostic: 'Posted-review state is ambiguous', + apiCalls: 1, + }, + { + name: 'marker-bearing review with active LGTM wording', + invocation: { + exitCode: '0', + verboseLog: 'no review', + reviews: [ + { + ...postedReview(), + body: `### Assessment: 🟢 NO FINDINGS\n\nLGTM!\n\n${TEST_MARKER}\n`, + }, + ], + }, + diagnostic: 'Posted-review state is ambiguous', + apiCalls: 1, + }, + { + name: 'marker-bearing review with active APPROVE wording', + invocation: { + exitCode: '124', + verboseLog: 'no review', + reviews: [ + { + ...postedReview(), + body: `### Assessment: 🟢 APPROVE\n\n${TEST_MARKER}\n`, + }, + ], + }, + diagnostic: 'Posted-review state is ambiguous', + apiCalls: 1, + }, + { + // A fresh same-SHA bot review without the marker means the template was + // bypassed or another integration posted mid-run — unattributable. + name: 'fresh same-SHA docker-agent review without a marker', + invocation: { exitCode: '0', verboseLog: 'no review', reviews: [unmarkedReview(101)] }, + diagnostic: 'Posted-review state is ambiguous', + apiCalls: 1, + }, + ])('fails closed on $name instead of trusting the exit code', ({ + invocation, + diagnostic, + apiCalls, + }) => { + const run = runSummary(invocation); + try { + expect(run.result.status).not.toBe(0); + expect(run.result.stderr).toContain(diagnostic); + expect(run.outputs).toContain('review-status=unverified'); + expect(run.records).toHaveLength(apiCalls); + expect(reviewCreations(run.records)).toEqual([]); + expect(run.summary).toContain('Review outcome unverified'); + expect(run.summary).not.toContain('Review completed'); + } finally { + rmSync(run.directory, { recursive: true, force: true }); + } + }); + + it('fails the step when the incomplete-review notice cannot be posted', () => { + // A silent no-post run whose notice also fails must surface as a failing + // step — never a warning-only false success — with an honest summary. + const run = runSummary({ exitCode: '0', verboseLog: 'no review', postFails: true }); + try { + expect(run.result.status).not.toBe(0); + expect(run.result.stdout).toContain('Failed to post the incomplete-review notice'); + const creations = reviewCreations(run.records); + expect(creations).toHaveLength(1); + expectReviewPayload(creations[0], summarySha, '⚠️ \\*\\*Review incomplete\\*\\*'); + expect(run.outputs).toContain('review-status=incomplete'); + expect(run.summary).toContain('Review incomplete'); + expect(run.summary).not.toContain('✅'); + } finally { + rmSync(run.directory, { recursive: true, force: true }); + } + }); + + it.each([ + { + name: 'timeout notice', + invocation: { exitCode: '124', verboseLog: 'no review', postFails: true }, + status: 'timed-out', + bodyPrefix: '⏱️', + diagnostic: 'Failed to post the timeout notice', + summaryStatus: '⏱️ **Review timed out**', + }, + { + name: 'failure notice', + invocation: { exitCode: '1', verboseLog: 'no review', postFails: true }, + status: 'failed', + bodyPrefix: '❌', + diagnostic: 'Failed to post the failure notice', + summaryStatus: '❌ **Review failed**', + }, + ])('fails the step when the $name cannot be posted', ({ + invocation, + status, + bodyPrefix, + diagnostic, + summaryStatus, + }) => { + // A no-post run whose fallback notice also fails must hard-fail like the + // incomplete-notice path, while keeping the honest status and summary so + // the completion reaction stays confused. + const run = runSummary(invocation); + try { + expect(run.result.status).not.toBe(0); + expect(run.result.stdout).toContain(diagnostic); + const creations = reviewCreations(run.records); + expect(creations).toHaveLength(1); + expectReviewPayload(creations[0], summarySha, bodyPrefix); + expect(run.outputs).toContain(`review-status=${status}`); + expect(run.summary).toContain(summaryStatus); + expect(run.summary).not.toContain('✅'); + } finally { + rmSync(run.directory, { recursive: true, force: true }); + } + }); + + it('reports a coherent partial success when the timeout hit after this run posted', () => { + const run = runSummary({ exitCode: '124', verboseLog: 'no review', reviews: [postedReview()] }); + try { + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.outputs).toContain('review-status=completed-with-warnings'); + expect(reviewCreations(run.records)).toEqual([]); + expect(run.summary).toContain('Review completed with warnings'); + expect(run.summary).not.toContain('**Review timed out**'); } finally { rmSync(run.directory, { recursive: true, force: true }); } @@ -1259,19 +1829,79 @@ describe('fork workflow security regressions', () => { }); it.each([ - ['missing reference', undefined], - ['retained template marker', 'jq -n --arg commit_id "__PR_HEAD_SHA__"'], - ['retained shell marker', 'jq -n --arg commit_id "$PR_HEAD_SHA"'], - ['zero commit argument', 'jq -n'], + ['missing reference', undefined, 'does not contain exactly one'], + [ + 'retained template marker', + 'jq -n --arg commit_id "__PR_HEAD_SHA__"', + 'does not contain exactly one', + ], + [ + 'retained shell marker', + 'jq -n --arg commit_id "$PR_HEAD_SHA"', + 'does not contain exactly one', + ], + ['zero commit argument', 'jq -n', 'does not contain exactly one'], [ 'multiple commit arguments', 'jq -n --arg commit_id "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" --arg commit_id "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"', + 'does not contain exactly one', ], [ 'selected/rendered SHA mismatch', 'jq -n --arg commit_id "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"', + 'does not contain exactly one', + ], + [ + 'missing posting route', + `jq -n --arg commit_id "${'a'.repeat(40)}"`, + 'does not route to exactly the trusted repository/PR', + ], + [ + 'literal route placeholders', + `jq -n --arg commit_id "${'a'.repeat(40)}" | gh api repos/{owner}/{repo}/pulls/{pr}/reviews --input -`, + 'does not route to exactly the trusted repository/PR', + ], + [ + 'retained repository marker', + `jq -n --arg commit_id "${'a'.repeat(40)}" | gh api "repos/__REPOSITORY__/pulls/88/reviews" --input -`, + 'does not route to exactly the trusted repository/PR', + ], + [ + 'untrusted hardcoded route', + `jq -n --arg commit_id "${'a'.repeat(40)}" | gh api "repos/evil/elsewhere/pulls/1/reviews" --input -`, + 'does not route to exactly the trusted repository/PR', ], - ])('executes malformed posting reference %s without API access', (_name, content) => { + [ + 'duplicate posting routes', + `jq -n --arg commit_id "${'a'.repeat(40)}"\ngh api "repos/docker/docker-agent-action/pulls/88/reviews" --input -\ngh api "repos/docker/docker-agent-action/pulls/88/reviews" --input -`, + 'does not route to exactly the trusted repository/PR', + ], + [ + 'missing finalize-body invocation', + `jq -n --arg commit_id "${'a'.repeat(40)}"\ngh api "repos/docker/docker-agent-action/pulls/88/reviews" --input -`, + "does not carry exactly this run's finalize-body invocation", + ], + [ + 'retained nonce placeholder', + `node /tmp/review-assessment.js finalize-body /tmp/review_body.md __REVIEW_RUN_NONCE__ /tmp/review_comments.json\njq -n --arg commit_id "${'a'.repeat(40)}"\ngh api "repos/docker/docker-agent-action/pulls/88/reviews" --input -`, + "does not carry exactly this run's finalize-body invocation", + ], + [ + 'stale nonce in the finalize invocation', + `node /tmp/review-assessment.js finalize-body /tmp/review_body.md ${'f'.repeat(32)} /tmp/review_comments.json\njq -n --arg commit_id "${'a'.repeat(40)}"\ngh api "repos/docker/docker-agent-action/pulls/88/reviews" --input -`, + "does not carry exactly this run's finalize-body invocation", + ], + [ + 'finalize invocation without the staged comments file', + `node /tmp/review-assessment.js finalize-body /tmp/review_body.md ${TEST_NONCE}\njq -n --arg commit_id "${'a'.repeat(40)}"\ngh api "repos/docker/docker-agent-action/pulls/88/reviews" --input -`, + "does not carry exactly this run's finalize-body invocation", + ], + [ + 'duplicate finalize invocations', + `node /tmp/review-assessment.js finalize-body /tmp/review_body.md ${TEST_NONCE} /tmp/review_comments.json\nnode /tmp/review-assessment.js finalize-body /tmp/review_body.md ${TEST_NONCE} /tmp/review_comments.json\njq -n --arg commit_id "${'a'.repeat(40)}"\ngh api "repos/docker/docker-agent-action/pulls/88/reviews" --input -`, + "does not carry exactly this run's finalize-body invocation", + ], + ])('executes malformed posting reference %s without API access', (_name, content, diagnostic) => { const directory = mkdtempSync(resolve(tmpdir(), 'docker-agent-reference-')); const reference = resolve(directory, 'posting-format.md'); if (content) writeFileSync(reference, content); @@ -1282,9 +1912,7 @@ describe('fork workflow security regressions', () => { }); try { expect(run.result.status).not.toBe(0); - expect(run.result.stderr).toContain( - 'Rendered posting reference does not contain exactly one', - ); + expect(run.result.stderr).toContain(diagnostic); expect(run.records).toEqual([]); } finally { rmSync(run.directory, { recursive: true, force: true }); @@ -1292,19 +1920,43 @@ describe('fork workflow security regressions', () => { } }); - it.each([ - { skipReason: 'concurrent' }, - { exitCode: '' }, - ])('executes benign skip states without requiring preflight inputs', (vector) => { + it('executes the concurrent-lock skip without requiring preflight inputs', () => { const directory = mkdtempSync(resolve(tmpdir(), 'docker-agent-skip-')); const run = runSummary({ - ...vector, + skipReason: 'concurrent', + exitCode: '', headSha: '', postingReference: resolve(directory, 'missing'), }); try { expect(run.result.status, run.result.stderr).toBe(0); expect(run.summary).toContain('Review skipped'); + expect(run.outputs).toContain('review-status=skipped'); + expect(run.records).toEqual([]); + } finally { + rmSync(run.directory, { recursive: true, force: true }); + rmSync(directory, { recursive: true, force: true }); + } + }); + + it('fails the composite no-exit path as setup-failed instead of labeling it skipped', () => { + // A setup step crashing before Run PR Review leaves EXIT_CODE empty with + // no intentional skip recorded. That must surface as a failing step with + // an honest non-skip status — never the neutral "skipped" (the mislabeled + // setup-failure regression). Only the concurrent lock may report skipped. + const directory = mkdtempSync(resolve(tmpdir(), 'docker-agent-setup-failed-')); + const run = runSummary({ + exitCode: '', + headSha: '', + postingReference: resolve(directory, 'missing'), + }); + try { + expect(run.result.status).not.toBe(0); + expect(run.result.stderr).toContain('a setup step failed before the review'); + expect(run.outputs).toContain('review-status=setup-failed'); + expect(run.outputs).not.toContain('review-status=skipped'); + expect(run.summary).toContain('Review setup failed'); + expect(run.summary).not.toContain('Review skipped'); expect(run.records).toEqual([]); } finally { rmSync(run.directory, { recursive: true, force: true }); @@ -1312,24 +1964,690 @@ describe('fork workflow security regressions', () => { } }); + const stagingTemplate = [ + 'jq -n --arg commit_id "__PR_HEAD_SHA__"', + 'node /tmp/review-assessment.js finalize-body /tmp/review_body.md __REVIEW_RUN_NONCE__ /tmp/review_comments.json', + 'gh api "repos/__REPOSITORY__/pulls/__PR_NUMBER__/reviews" --input -', + ].join('\n'); + it.each([ - ['valid immutable SHA', 'a'.repeat(40), 'jq -n --arg commit_id "__PR_HEAD_SHA__"'], - ['empty SHA', '', 'jq -n --arg commit_id "__PR_HEAD_SHA__"'], - ['non-hex SHA', 'g'.repeat(40), 'jq -n --arg commit_id "__PR_HEAD_SHA__"'], - ['short SHA', 'a'.repeat(39), 'jq -n --arg commit_id "__PR_HEAD_SHA__"'], - ['long SHA', 'a'.repeat(41), 'jq -n --arg commit_id "__PR_HEAD_SHA__"'], - ['unresolved template', 'a'.repeat(40), 'jq -n --arg commit_id "$PR_HEAD_SHA"'], - ['zero commit arguments', 'a'.repeat(40), 'jq -n --arg body "review"'], - [ - 'multiple commit arguments', - 'a'.repeat(40), - 'jq -n --arg commit_id "__PR_HEAD_SHA__" --arg commit_id "x"', - ], - ])('executes Copy reference files staging preflight for %s', (_name, sha, template) => { - const result = runCopyReference(sha, template); - expect(result.status, result.stderr).toBe( - template === 'jq -n --arg commit_id "__PR_HEAD_SHA__"' && /^[a-f0-9]{40}$/i.test(sha) ? 0 : 1, + { name: 'valid immutable SHA and trusted route', ok: true }, + { name: 'empty SHA', sha: '', ok: false }, + { name: 'non-hex SHA', sha: 'g'.repeat(40), ok: false }, + { name: 'short SHA', sha: 'a'.repeat(39), ok: false }, + { name: 'long SHA', sha: 'a'.repeat(41), ok: false }, + { + name: 'unresolved template', + template: stagingTemplate.replace('__PR_HEAD_SHA__', '$PR_HEAD_SHA'), + ok: false, + }, + { + name: 'zero commit arguments', + template: `jq -n --arg body "review"\ngh api "repos/__REPOSITORY__/pulls/__PR_NUMBER__/reviews" --input -`, + ok: false, + }, + { + name: 'multiple commit arguments', + template: `${stagingTemplate} --arg commit_id "x"`, + ok: false, + }, + { + name: 'missing posting route', + template: 'jq -n --arg commit_id "__PR_HEAD_SHA__"', + ok: false, + }, + { + name: 'literal route placeholders', + template: + 'jq -n --arg commit_id "__PR_HEAD_SHA__"\ngh api repos/{owner}/{repo}/pulls/{pr}/reviews --input -', + ok: false, + }, + { + name: 'untrusted hardcoded route', + template: + 'jq -n --arg commit_id "__PR_HEAD_SHA__"\ngh api "repos/evil/elsewhere/pulls/1/reviews" --input -', + ok: false, + }, + { + name: 'duplicate posting routes', + template: `${stagingTemplate}\ngh api "repos/__REPOSITORY__/pulls/__PR_NUMBER__/reviews" --input -`, + ok: false, + }, + { + name: 'missing finalize-body invocation', + template: + 'jq -n --arg commit_id "__PR_HEAD_SHA__"\ngh api "repos/__REPOSITORY__/pulls/__PR_NUMBER__/reviews" --input -', + ok: false, + }, + { + name: 'finalize-body without the staged comments file', + template: + 'jq -n --arg commit_id "__PR_HEAD_SHA__"\nnode /tmp/review-assessment.js finalize-body /tmp/review_body.md __REVIEW_RUN_NONCE__\ngh api "repos/__REPOSITORY__/pulls/__PR_NUMBER__/reviews" --input -', + ok: false, + }, + { + name: 'duplicate finalize-body invocations', + template: `node /tmp/review-assessment.js finalize-body /tmp/review_body.md __REVIEW_RUN_NONCE__ /tmp/review_comments.json\n${stagingTemplate}`, + ok: false, + }, + { + name: 'hardcoded foreign nonce in the template', + template: stagingTemplate.replace('__REVIEW_RUN_NONCE__', 'f'.repeat(32)), + ok: false, + }, + { name: 'repository without owner', repository: 'no-slash-repo', ok: false }, + { name: 'repository with sed metacharacters', repository: 'docker/repo|x', ok: false }, + { name: 'repository with replacement metacharacter', repository: 'docker/re&po', ok: false }, + { name: 'non-numeric PR number', prNumber: '88x', ok: false }, + { name: 'empty PR number', prNumber: '', ok: false }, + { name: 'empty run nonce', nonce: '', ok: false }, + { name: 'malformed run nonce', nonce: 'abc-123', ok: false }, + { name: 'uppercase run nonce', nonce: TEST_NONCE.toUpperCase(), ok: false }, + ])('executes Copy reference files staging preflight for $name', ({ + sha, + template, + repository, + prNumber, + nonce, + ok, + }) => { + const headSha = sha ?? 'a'.repeat(40); + const { result, rendered } = runCopyReference( + headSha, + template ?? stagingTemplate, + repository, + prNumber, + nonce, ); + expect(result.status, result.stderr).toBe(ok ? 0 : 1); + if (ok) { + expect(rendered).toContain(`--arg commit_id "${headSha}"`); + expect(rendered).toContain( + 'gh api "repos/docker/docker-agent-action/pulls/88/reviews" --input -', + ); + expect(rendered).toContain( + `finalize-body /tmp/review_body.md ${TEST_NONCE} /tmp/review_comments.json`, + ); + expect(rendered).not.toMatch( + /__PR_HEAD_SHA__|__REPOSITORY__|__PR_NUMBER__|__REVIEW_RUN_NONCE__|\{owner\}/, + ); + } + }); + + it('renders the repository posting template with the trusted SHA, route, and nonce staged in', () => { + const sha = 'a'.repeat(40); + const { result, rendered } = runCopyReference( + sha, + readFileSync(resolve(root, 'review-pr/agents/refs/posting-format.md'), 'utf8'), + ); + expect(result.status, result.stderr).toBe(0); + expect(rendered).toContain(`--arg commit_id "${sha}"`); + expect(rendered).toContain( + '&& gh api "repos/docker/docker-agent-action/pulls/88/reviews" --input - < /tmp/review_payload.json', + ); + // No trusted routing data is left for the model to substitute. + expect(rendered).not.toMatch( + /__PR_HEAD_SHA__|__REPOSITORY__|__PR_NUMBER__|__REVIEW_RUN_NONCE__|\{owner\}|\{repo\}|\{pr\}/, + ); + // The review body is a heredoc-written file, validated and marker-stamped + // by the trusted CLI — no REVIEW_BODY shell variable exists to bleed a + // default assessment through, and the marker append is mechanical. + expect(rendered).not.toMatch(/^REVIEW_BODY=/m); + expect(rendered).not.toMatch(/\$REVIEW_BODY|\$\{REVIEW_BODY/); + expect(rendered).toContain('test -s /tmp/review_body.md \\'); + expect(rendered).toContain( + `&& node /tmp/review-assessment.js finalize-body /tmp/review_body.md ${TEST_NONCE} /tmp/review_comments.json \\`, + ); + expect(rendered).toContain('--rawfile body /tmp/review_body.md'); + // The payload is staged to a trusted temp file and validated before gh + // ever runs — a failed jq must not start the API call (the old `jq | gh` + // pipe launched gh regardless of jq's fate). + expect(rendered).toContain('> /tmp/review_payload.json \\'); + expect(rendered).toContain( + `&& jq -e 'type == "object"' /tmp/review_payload.json > /dev/null \\`, + ); + expect(rendered).not.toMatch(/\|\s*gh api/); + }); + + function extractPostingCommand(rendered: string): string { + const lines = rendered.split('\n'); + const start = lines.findIndex((line) => line.startsWith('test -s /tmp/review_body.md')); + const end = lines.findIndex( + (line, index) => index > start && line.trimStart().startsWith('&& gh api '), + ); + if (start === -1 || end === -1) + throw new Error('chained posting command not found in rendered template'); + return lines.slice(start, end + 1).join('\n'); + } + + it('executes the rendered posting command: invalid bodies refuse, computed outcome posts with the marker', () => { + const sha = 'a'.repeat(40); + const { result, rendered } = runCopyReference( + sha, + readFileSync(resolve(root, 'review-pr/agents/refs/posting-format.md'), 'utf8'), + ); + expect(result.status, result.stderr).toBe(0); + const command = extractPostingCommand(rendered); + const directory = mkdtempSync(resolve(tmpdir(), 'docker-agent-posting-guard-')); + try { + const recordsPath = resolve(directory, 'gh-records.jsonl'); + const comments = resolve(directory, 'review_comments.json'); + const body = resolve(directory, 'review_body.md'); + const payload = resolve(directory, 'review_payload.json'); + writeFileSync(resolve(directory, 'gh'), ghMock()); + chmodSync(resolve(directory, 'gh'), 0o755); + const spawnPosting = ( + bodyContent: string | undefined, + options: { comments?: string | null; env?: Record } = {}, + ) => { + writeFileSync(recordsPath, ''); + rmSync(body, { force: true }); + rmSync(comments, { force: true }); + rmSync(payload, { force: true }); + const commentsContent = options.comments === undefined ? '[]\n' : options.comments; + if (commentsContent !== null) writeFileSync(comments, commentsContent); + if (bodyContent !== undefined) writeFileSync(body, bodyContent); + writeFileSync( + resolve(directory, 'post.sh'), + command + .replaceAll('/tmp/review_comments.json', comments) + .replaceAll('/tmp/review_body.md', body) + .replaceAll('/tmp/review_payload.json', payload) + .replaceAll('/tmp/review-assessment.js', reviewAssessmentCli), + ); + return spawnSync('/bin/bash', ['--noprofile', '--norc', resolve(directory, 'post.sh')], { + cwd: directory, + env: testEnvironment({ + PATH: `${directory}${delimiter}${safePath}`, + GH_RECORDS: recordsPath, + ...options.env, + }), + encoding: 'utf8', + }); + }; + const ghRecords = () => + readFileSync(recordsPath, 'utf8') + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as GhRecord); + + // Missing and empty body files: `test -s` refuses before any gh call. + for (const content of [undefined, '']) { + const refused = spawnPosting(content); + expect(refused.status, String(content)).not.toBe(0); + expect(ghRecords()).toEqual([]); + } + + // Forbidden approval wording, missing status line, and 🟢 NO FINDINGS + // over findings sections: the trusted validator refuses posting. + for (const [content, reason] of [ + ['### Assessment: 🟢 NO FINDINGS\n\nLGTM!\n', 'forbidden approval wording'], + ['🟢 **No issues found** — all good.\n', 'forbidden approval wording'], + ['looks good to me\n', 'no recognized status line'], + ['### ⚠️ Review incomplete\n### Assessment: 🟢 NO FINDINGS\n', 'mixes an assessment line'], + [ + '### Assessment: 🟢 NO FINDINGS\n\n#### Low-severity findings (not verified, not posted inline)\n- [low] a.go:1 — x\n', + 'cannot be combined with findings sections', + ], + ] as const) { + const refused = spawnPosting(content); + expect(refused.status, content).not.toBe(0); + expect(refused.stderr, content).toContain(reason); + expect(ghRecords()).toEqual([]); + } + + // Missing, malformed, and non-array staged comments files: the trusted + // validator refuses before jq or gh ever run. + for (const [staged, reason] of [ + [null, 'missing or unreadable'], + ['not json', 'not valid JSON'], + ['{"body": "x"}', 'must be a JSON array'], + ] as const) { + const refused = spawnPosting('### Assessment: 🟡 NEEDS ATTENTION\n', { comments: staged }); + expect(refused.status, String(staged)).not.toBe(0); + expect(refused.stderr, String(staged)).toContain(reason); + expect(ghRecords()).toEqual([]); + } + + // 🟢 NO FINDINGS over staged inline comments contradicts the label — + // refused at runtime, zero gh calls. + const contradicted = spawnPosting('### Assessment: 🟢 NO FINDINGS\n', { + comments: '[{"path": "a.go", "line": 1, "body": "**[low] issue**"}]\n', + }); + expect(contradicted.status).not.toBe(0); + expect(contradicted.stderr).toContain('🟢 NO FINDINGS cannot be posted with 1 staged'); + expect(ghRecords()).toEqual([]); + + // Forced jq failure: payload staging fails, so gh is NEVER invoked — + // the old `jq | gh` pipe started gh regardless of jq's fate and relied + // on pipefail/API rejection to surface the error. + const brokenTools = resolve(directory, 'broken-tools'); + mkdirSync(brokenTools, { recursive: true }); + writeFileSync(resolve(brokenTools, 'jq'), '#!/bin/sh\nexit 7\n'); + chmodSync(resolve(brokenTools, 'jq'), 0o755); + const jqFailed = spawnPosting('### Assessment: 🟡 NEEDS ATTENTION\n', { + env: { PATH: `${brokenTools}${delimiter}${directory}${delimiter}${safePath}` }, + }); + expect(jqFailed.status).not.toBe(0); + expect(ghRecords()).toEqual([]); + + // gh itself failing propagates a nonzero chain exit. + const ghFailed = spawnPosting('### Assessment: 🟡 NEEDS ATTENTION\n', { + env: { GH_POST_FAILS: '1' }, + }); + expect(ghFailed.status).not.toBe(0); + expect(ghRecords()).toHaveLength(1); + + // With the computed outcome written, the chain validates, appends the + // run marker mechanically, and posts the exact hardcoded payload. + const posted = spawnPosting('### Assessment: 🟡 NEEDS ATTENTION\n'); + expect(posted.status, posted.stderr).toBe(0); + const records = ghRecords(); + expect(records).toHaveLength(1); + expect(records[0].args).toBe( + 'api repos/docker/docker-agent-action/pulls/88/reviews --input -', + ); + expect(JSON.parse(records[0].input)).toEqual({ + body: `### Assessment: 🟡 NEEDS ATTENTION\n\n${TEST_MARKER}\n`, + event: 'COMMENT', + commit_id: sha, + comments: [], + }); + + // Staged inline comments ride along for non-NO-FINDINGS outcomes. + const withComments = spawnPosting('### Assessment: 🟡 NEEDS ATTENTION\n', { + comments: '[{"path": "a.go", "line": 1, "body": "**[low] issue**"}]\n', + }); + expect(withComments.status, withComments.stderr).toBe(0); + const commentRecords = ghRecords(); + expect(commentRecords).toHaveLength(1); + expect(JSON.parse(commentRecords[0].input).comments).toEqual([ + { path: 'a.go', line: 1, body: '**[low] issue**' }, + ]); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + function runBaseline(options: { reviews?: string; fetchFails?: boolean }): { + result: ReturnType; + outputs: string; + } { + const directory = mkdtempSync(resolve(tmpdir(), 'docker-agent-baseline-')); + try { + const output = resolve(directory, 'output'); + writeFileSync(output, ''); + writeFileSync(resolve(directory, 'gh-records.jsonl'), ''); + writeFileSync(resolve(directory, 'reviews.json'), options.reviews ?? '[]'); + writeFileSync(resolve(directory, 'baseline.sh'), actionStepRun('Capture review baseline')); + writeFileSync(resolve(directory, 'gh'), ghMock()); + chmodSync(resolve(directory, 'gh'), 0o755); + const result = spawnSync( + '/bin/bash', + ['--noprofile', '--norc', '-e', '-o', 'pipefail', resolve(directory, 'baseline.sh')], + { + cwd: directory, + env: testEnvironment({ + PATH: `${directory}${delimiter}${safePath}`, + GH_RECORDS: resolve(directory, 'gh-records.jsonl'), + GH_REVIEWS_FILE: resolve(directory, 'reviews.json'), + GH_REVIEWS_FETCH_FAILS: options.fetchFails ? '1' : '', + GITHUB_OUTPUT: output, + REPOSITORY: 'docker/docker-agent-action', + PR_NUMBER: '88', + }), + encoding: 'utf8', + }, + ); + return { result, outputs: readFileSync(output, 'utf8') }; + } finally { + rmSync(directory, { recursive: true, force: true }); + } + } + + it.each([ + { name: 'existing reviews', reviews: '[{"id": 7}, {"id": 12}]', max: '12' }, + { name: 'no reviews', reviews: '[]', max: '0' }, + { name: 'null review IDs', reviews: '[{"id": null}]', max: '0' }, + ])('executes the review baseline capture for $name', ({ reviews, max }) => { + const { result, outputs } = runBaseline({ reviews }); + expect(result.status, result.stderr).toBe(0); + expect(outputs).toContain(`max-review-id=${max}\n`); + }); + + it.each([ + { name: 'lookup failure', options: { fetchFails: true } }, + { name: 'non-JSON payload', options: { reviews: 'not json' } }, + ])('refuses to start an unverifiable review on baseline $name', ({ options }) => { + const { result, outputs } = runBaseline(options); + expect(result.status).not.toBe(0); + expect(result.stdout + result.stderr).toContain('refusing to start an unverifiable review'); + expect(outputs).not.toContain('max-review-id='); + }); + + function runNonceGeneration(cliSource?: string): { + result: ReturnType; + outputs: string; + } { + const directory = mkdtempSync(resolve(tmpdir(), 'docker-agent-nonce-')); + try { + const actionPath = resolve(directory, 'action'); + mkdirSync(actionPath, { recursive: true }); + mkdirSync(resolve(directory, 'dist'), { recursive: true }); + if (cliSource === undefined) { + cpSync(reviewAssessmentCli, resolve(directory, 'dist/review-assessment.js')); + } else { + writeFileSync(resolve(directory, 'dist/review-assessment.js'), cliSource); + } + const output = resolve(directory, 'output'); + writeFileSync(output, ''); + writeFileSync( + resolve(directory, 'nonce.sh'), + actionStepRun('Generate run attribution nonce'), + ); + const result = spawnSync( + '/bin/bash', + ['--noprofile', '--norc', '-e', '-o', 'pipefail', resolve(directory, 'nonce.sh')], + { + env: testEnvironment({ ACTION_PATH: actionPath, GITHUB_OUTPUT: output }), + encoding: 'utf8', + }, + ); + return { result, outputs: readFileSync(output, 'utf8') }; + } finally { + rmSync(directory, { recursive: true, force: true }); + } + } + + it('executes the nonce generation step masking the nonce before any other output', () => { + const { result, outputs } = runNonceGeneration(); + expect(result.status, result.stderr).toBe(0); + const nonce = outputs.match(/^nonce=([0-9a-f]{32})$/m)?.[1]; + expect(nonce).toBeDefined(); + if (!nonce) throw new Error('nonce output missing'); + // The FIRST line the step prints is the ::add-mask:: workflow command, so + // the runner masks the nonce before any later step (or this one) can echo + // it into the public run log. + const stdout = String(result.stdout); + expect(stdout.split('\n')[0]).toBe(`::add-mask::${nonce}`); + // Outside the masking command the nonce never appears in normal output. + expect(stdout.replace(`::add-mask::${nonce}`, '')).not.toContain(nonce); + expect(String(result.stderr)).not.toContain(nonce); + // Static ordering: the run body masks immediately after generation, before + // the validation guard and before the value reaches GITHUB_OUTPUT. + const run = actionStepRun('Generate run attribution nonce'); + const mask = run.indexOf('echo "::add-mask::$RUN_NONCE"'); + expect(mask).toBeGreaterThan(run.indexOf('new-run-nonce')); + expect(mask).toBeLessThan(run.indexOf('[[ "$RUN_NONCE" =~')); + expect(mask).toBeLessThan(run.indexOf('GITHUB_OUTPUT')); + }); + + it('still masks and refuses staging when the generated nonce is malformed', () => { + const { result, outputs } = runNonceGeneration('process.stdout.write("not-a-nonce\\n");\n'); + expect(result.status).not.toBe(0); + expect(result.stdout + result.stderr).toContain('nonce is malformed'); + expect(outputs).not.toContain('nonce='); + expect(String(result.stdout).split('\n')[0]).toBe('::add-mask::not-a-nonce'); + }); + + function runReaction(reviewStatus: string): { + result: ReturnType; + records: GhRecord[]; + } { + const directory = mkdtempSync(resolve(tmpdir(), 'docker-agent-reaction-')); + try { + const recordsPath = resolve(directory, 'gh-records.jsonl'); + writeFileSync(recordsPath, ''); + writeFileSync(resolve(directory, 'reaction.sh'), actionStepRun('Add completion reaction')); + writeFileSync(resolve(directory, 'gh'), ghMock()); + chmodSync(resolve(directory, 'gh'), 0o755); + const result = spawnSync( + '/bin/bash', + ['--noprofile', '--norc', '-e', '-o', 'pipefail', resolve(directory, 'reaction.sh')], + { + cwd: directory, + env: testEnvironment({ + PATH: `${directory}${delimiter}${safePath}`, + GH_RECORDS: recordsPath, + REVIEW_STATUS: reviewStatus, + REPO: 'docker/docker-agent-action', + COMMENT_ID: '55', + }), + encoding: 'utf8', + }, + ); + const records = readFileSync(recordsPath, 'utf8') + .trim() + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as GhRecord); + return { result, records }; + } finally { + rmSync(directory, { recursive: true, force: true }); + } + } + + it.each([ + ['completed', '+1'], + ['completed-with-warnings', '+1'], + ['incomplete', 'confused'], + ['inconclusive', 'confused'], + ['failed', 'confused'], + ['timed-out', 'confused'], + ['skipped', 'confused'], + ['setup-failed', 'confused'], + ['unverified', 'confused'], + ['', 'confused'], + ])('executes the completion reaction for review-status %j as %s', (status, reaction) => { + // An exit-0 run that posted nothing carries review-status=incomplete and + // must react confused — never 👍 (the false-success reaction regression). + const run = runReaction(status); + expect(run.result.status, run.result.stderr).toBe(0); + expect(run.records).toHaveLength(1); + expect(run.records[0].args).toBe( + `api repos/docker/docker-agent-action/issues/comments/55/reactions -X POST -f content=${reaction}`, + ); + }); + + function runEnforceOutcome( + reviewStatus: string, + stepOutcome: string, + ): ReturnType { + const directory = mkdtempSync(resolve(tmpdir(), 'docker-agent-enforce-')); + try { + const body = step('review', 'Enforce review outcome').run; + if (!body) throw new Error('Expected an Enforce review outcome run body'); + writeFileSync(resolve(directory, 'enforce.sh'), body); + return spawnSync( + '/bin/bash', + ['--noprofile', '--norc', '-e', '-o', 'pipefail', resolve(directory, 'enforce.sh')], + { + env: testEnvironment({ REVIEW_STATUS: reviewStatus, RUN_REVIEW_OUTCOME: stepOutcome }), + encoding: 'utf8', + }, + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + } + + it.each([ + // Only a verified completion or an intentional skip leaves the job green. + ['completed', 'success', 0], + ['completed-with-warnings', 'success', 0], + ['skipped', 'success', 0], + // The review step never ran: a pre-review guard (draft, auth, rate + // anomaly, non-/review comment) filtered the event — an expected skip. + ['', 'skipped', 0], + // A missing status from a step that ran means the composite crashed + // before the summary — never a green outcome. + ['', 'success', 1], + ['', 'failure', 1], + // A skipped status from a FAILED composite is a contradiction — the + // failure wins (a mislabeled setup failure must never look intentional). + ['skipped', 'failure', 1], + // continue-on-error masks these from job.status; the gate restores them. + ['incomplete', 'success', 1], + ['inconclusive', 'success', 1], + ['failed', 'success', 1], + ['timed-out', 'success', 1], + ['setup-failed', 'failure', 1], + ['unverified', 'failure', 1], + ] as [ + string, + string, + number, + ][])('executes the review outcome enforcement for status %j (step outcome %j) with exit %i', (status, outcome, exit) => { + const result = runEnforceOutcome(status, outcome); + expect(result.status, result.stdout + result.stderr).toBe(exit); + if (exit !== 0) { + expect(result.stdout + result.stderr).toContain('failing the review job'); + } + }); + + it('runs the outcome enforcement gate last, on every path, off the review-status output', () => { + const enforce = step('review', 'Enforce review outcome'); + expect(enforce.if).toBe('always()'); + expect(enforce.env?.REVIEW_STATUS).toBe('${' + '{ steps.run-review.outputs.review-status }}'); + expect(enforce.env?.RUN_REVIEW_OUTCOME).toBe('${' + '{ steps.run-review.outcome }}'); + // Last step of the job: the cleanup/check-update steps run before the + // gate can fail the job. + const steps = job('review').steps ?? []; + expect(steps[steps.length - 1]?.name).toBe('Enforce review outcome'); + expect(stepIndex('review', 'Enforce review outcome')).toBeGreaterThan( + stepIndex('review', 'Update check run'), + ); + // The reusable workflow re-exposes the API-verified status to callers. + expect(job('review').outputs?.['review-status']).toBe( + '${' + '{ steps.run-review.outputs.review-status }}', + ); + const workflowCall = workflow.on?.workflow_call as { + outputs?: Record; + }; + expect(workflowCall.outputs?.['review-status']?.value).toBe( + '${' + '{ jobs.review.outputs.review-status }}', + ); + }); + + it('executes the check-run conclusion script keyed on the API-verified review-status', async () => { + const check = step('review', 'Update check run'); + expect(check.if).toBe("always() && steps.create-check.outputs.check-id != ''"); + expect(check.env?.REVIEW_STATUS).toBe('${' + '{ steps.run-review.outputs.review-status }}'); + expect(check.env?.RUN_REVIEW_OUTCOME).toBe('${' + '{ steps.run-review.outcome }}'); + const script = check.with?.script; + if (!script) throw new Error('Expected an Update check run script'); + const AsyncFunction = (async () => {}).constructor as new ( + ...args: string[] + ) => (github: unknown, context: unknown, core: unknown) => Promise; + const runScript = new AsyncFunction('github', 'context', 'core', script); + const envKeys = ['CHECK_ID', 'JOB_STATUS', 'REVIEW_STATUS', 'RUN_REVIEW_OUTCOME'] as const; + + const conclusionFor = async (env: Record): Promise => { + const updates: Array> = []; + const github = { + rest: { + checks: { + update: async (args: Record) => { + updates.push(args); + }, + }, + }, + }; + const saved = envKeys.map((key) => [key, process.env[key]] as const); + Object.assign(process.env, { + CHECK_ID: '7', + JOB_STATUS: 'success', + REVIEW_STATUS: '', + RUN_REVIEW_OUTCOME: '', + ...env, + }); + try { + await runScript( + github, + { repo: { owner: 'docker', repo: 'docker-agent-action' } }, + { warning: () => {} }, + ); + } finally { + for (const [key, value] of saved) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + expect(updates).toHaveLength(1); + expect(updates[0].check_run_id).toBe(7); + expect(updates[0].status).toBe('completed'); + return updates[0].conclusion; + }; + + expect(await conclusionFor({ JOB_STATUS: 'cancelled' })).toBe('cancelled'); + expect(await conclusionFor({ REVIEW_STATUS: 'completed' })).toBe('success'); + expect(await conclusionFor({ REVIEW_STATUS: 'completed-with-warnings' })).toBe('success'); + // Intentional skips stay neutral — including the guard-skipped step — but + // ONLY when the composite itself did not fail: a failed step claiming + // skipped must never yield a neutral check. + expect(await conclusionFor({ REVIEW_STATUS: 'skipped' })).toBe('neutral'); + expect(await conclusionFor({ REVIEW_STATUS: 'skipped', RUN_REVIEW_OUTCOME: 'success' })).toBe( + 'neutral', + ); + expect(await conclusionFor({ REVIEW_STATUS: 'skipped', RUN_REVIEW_OUTCOME: 'failure' })).toBe( + 'failure', + ); + expect(await conclusionFor({ RUN_REVIEW_OUTCOME: 'skipped' })).toBe('neutral'); + // Everything else — non-success statuses and a missing status from a + // crashed composite — is red, even though continue-on-error keeps + // job.status green (the misleading-green regression). + for (const status of [ + 'incomplete', + 'inconclusive', + 'failed', + 'timed-out', + 'setup-failed', + 'unverified', + ]) { + expect(await conclusionFor({ REVIEW_STATUS: status }), status).toBe('failure'); + } + expect(await conclusionFor({ RUN_REVIEW_OUTCOME: 'success' })).toBe('failure'); + }); + + it('captures the API review baseline before the agent runs and keys the reaction on review-status', () => { + const names = reviewActionSteps().map((candidate) => candidate.name); + const baseline = names.indexOf('Capture review baseline'); + expect(baseline).toBeGreaterThan(names.indexOf('Fetch existing review comments')); + expect(baseline).toBeLessThan(names.indexOf('Run PR Review')); + expect(reviewActionStep('Capture review baseline').if).toBe( + "steps.lock-check.outputs.skip != 'true'", + ); + // The attribution nonce is generated in a trusted step before the posting + // template is staged, and both the staging step and the summary consume + // exactly that output — never a user-controllable value. + const nonce = names.indexOf('Generate run attribution nonce'); + expect(nonce).toBeGreaterThan(-1); + expect(nonce).toBeLessThan(names.indexOf('Copy reference files')); + expect(reviewActionStep('Generate run attribution nonce').if).toBe( + "steps.lock-check.outputs.skip != 'true'", + ); + expect(reviewActionStep('Copy reference files').env?.RUN_NONCE).toBe( + '${' + '{ steps.run-nonce.outputs.nonce }}', + ); + const summary = reviewActionStep('Post clean summary'); + expect(summary.env?.BASELINE_MAX_REVIEW_ID).toBe( + '${' + '{ steps.review-baseline.outputs.max-review-id }}', + ); + expect(summary.env?.RUN_NONCE).toBe('${' + '{ steps.run-nonce.outputs.nonce }}'); + // Post-run attribution runs through the bundled review-assessment CLI + // (exact marker + selected SHA + baseline), not inline jq identity + // filters — the same logic the unit tests pin. + expect(summary.run).toContain('dist/review-assessment.js" classify-run'); + expect(summary.run).toContain('"$PR_HEAD_SHA" "$BASELINE_MAX_REVIEW_ID" "$RUN_NONCE"'); + expect(summary.run).not.toContain('AGENT_REVIEWS_ON_SHA'); + const reaction = reviewActionStep('Add completion reaction'); + expect(reaction.env?.REVIEW_STATUS).toBe( + '${' + '{ steps.post-summary.outputs.review-status }}', + ); + // The reaction must key on the API-verified status, never the exit code. + expect(reaction.env?.EXIT_CODE).toBeUndefined(); + expect(reaction.run).not.toContain('EXIT_CODE'); }); it('binds immutable review inputs before the snapshot and derives posting from its output', () => { @@ -1346,6 +2664,27 @@ describe('fork workflow security regressions', () => { expect(summary).toContain(`PR_HEAD_SHA: ${'${'}{ steps.pr-info.outputs.head-sha }}`); }); + it('keeps the review action free of synthesized approvals and low-finding suppression', () => { + const action = readFileSync(resolve(root, 'review-pr/action.yml'), 'utf8'); + // The prompt must not tell the agent to report only verified findings — + // that wording suppressed unverified low findings (docker/gordon #1814). + expect(action).not.toContain( + 'Only report CONFIRMED and LIKELY findings. Always post as COMMENT', + ); + expect(action).toContain( + 'Surviving low-severity findings skip verification but MUST still be surfaced', + ); + // The prompt pins the COMMENT event and the neutral zero-findings label; + // no fallback may pass an approving event to the Reviews API. + expect(action).toContain('Always post as COMMENT (never APPROVE or REQUEST_CHANGES)'); + expect(action).toContain('--arg event "COMMENT"'); + expect(action).not.toMatch(/--arg event "(?:APPROVE|REQUEST_CHANGES)"/); + expect(action).not.toContain('🟢 APPROVE'); + // No code path may synthesize an LGTM/no-issues review body. + expect(action).not.toContain('LGTM!'); + expect(action).not.toContain('🟢 **No issues found**'); + }); + it('keeps resolver output names body-free and shell expressions out of run bodies', () => { const outputs = resolverOutputNames(); expect(outputs).toContain('comment-in-reply-to-id'); diff --git a/src/review-assessment/__tests__/build-cli.ts b/src/review-assessment/__tests__/build-cli.ts new file mode 100644 index 0000000..0f1a55d --- /dev/null +++ b/src/review-assessment/__tests__/build-cli.ts @@ -0,0 +1,39 @@ +// Copyright The Docker Agent Action authors +// SPDX-License-Identifier: Apache-2.0 + +import { existsSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { build } from 'tsup'; + +let cliPromise: Promise | null = null; + +/** + * Bundle the review-assessment CLI the same way `pnpm build` does + * (tsup/esbuild, ESM, self-contained) so shell harnesses can execute the + * exact runtime artifact with plain `node`. Built once per test process into + * a throwaway directory; the module has no npm dependencies, so the build is + * fast and needs no banner/define plumbing from the main tsup config. + */ +export function builtReviewAssessmentCli(): Promise { + cliPromise ??= (async () => { + const outDir = mkdtempSync(join(tmpdir(), 'review-assessment-dist-')); + await build({ + config: false, + entry: { 'review-assessment': resolve(import.meta.dirname, '../index.ts') }, + format: ['esm'], + platform: 'node', + target: 'node24', + outDir, + outExtension: () => ({ js: '.js' }), + sourcemap: false, + clean: false, + splitting: false, + silent: true, + }); + const cli = join(outDir, 'review-assessment.js'); + if (!existsSync(cli)) throw new Error('review-assessment CLI build produced no output'); + return cli; + })(); + return cliPromise; +} diff --git a/src/review-assessment/__tests__/index.test.ts b/src/review-assessment/__tests__/index.test.ts new file mode 100644 index 0000000..862efa5 --- /dev/null +++ b/src/review-assessment/__tests__/index.test.ts @@ -0,0 +1,147 @@ +// Copyright The Docker Agent Action authors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Unit tests for the review-assessment CLI wiring (main with mocked exit). + * + * The full shell-level surface (new-run-nonce → sed staging → finalize-body → + * classify-run inside the action's scripts) is executed end-to-end by the + * built-CLI harness in src/resolve-trigger-context/__tests__/ + * workflow-security.test.ts; these tests pin the argument validation and file + * effects at the module boundary. + */ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { main, newRunNonce } from '../index.js'; + +const NONCE = '0123456789abcdef0123456789abcdef'; +const MARKER = ``; + +let directory: string; +let stdout: string[]; +let errors: string[]; + +beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), 'review-assessment-cli-')); + stdout = []; + errors = []; + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + stdout.push(String(chunk)); + return true; + }); + vi.spyOn(console, 'error').mockImplementation((message) => { + errors.push(String(message)); + }); + vi.spyOn(process, 'exit').mockImplementation((code) => { + throw new Error(`exit:${code}`); + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); + rmSync(directory, { recursive: true, force: true }); +}); + +describe('new-run-nonce', () => { + it('prints a fresh 32-hex nonce per invocation', () => { + main(['new-run-nonce']); + main(['new-run-nonce']); + const [first, second] = stdout.map((line) => line.trim()); + expect(first).toMatch(/^[0-9a-f]{32}$/); + expect(second).toMatch(/^[0-9a-f]{32}$/); + expect(first).not.toBe(second); + }); + + it('generates unique unguessable nonces', () => { + const nonces = new Set(Array.from({ length: 64 }, () => newRunNonce())); + expect(nonces.size).toBe(64); + }); +}); + +describe('finalize-body', () => { + function stageComments(content: string, name = 'review_comments.json'): string { + const commentsFile = join(directory, name); + writeFileSync(commentsFile, content); + return commentsFile; + } + + it('validates the body against the staged comments and appends the run marker in place', () => { + const bodyFile = join(directory, 'review_body.md'); + writeFileSync(bodyFile, '### Assessment: 🟢 NO FINDINGS\n'); + const commentsFile = stageComments('[]\n'); + main(['finalize-body', bodyFile, NONCE, commentsFile]); + expect(readFileSync(bodyFile, 'utf8')).toBe(`### Assessment: 🟢 NO FINDINGS\n\n${MARKER}\n`); + // Idempotent: a retried posting command must not double-append. + main(['finalize-body', bodyFile, NONCE, commentsFile]); + expect(readFileSync(bodyFile, 'utf8')).toBe(`### Assessment: 🟢 NO FINDINGS\n\n${MARKER}\n`); + }); + + it('accepts staged inline comments for a non-NO-FINDINGS outcome', () => { + const bodyFile = join(directory, 'review_body.md'); + writeFileSync(bodyFile, '### Assessment: 🟡 NEEDS ATTENTION\n'); + const commentsFile = stageComments('[{"path": "a.go", "line": 1, "body": "**[low] x**"}]\n'); + main(['finalize-body', bodyFile, NONCE, commentsFile]); + expect(readFileSync(bodyFile, 'utf8')).toContain(MARKER); + }); + + it.each([ + ['missing file', join('nowhere', 'review_body.md'), NONCE, undefined], + ['empty body', 'empty.md', NONCE, ''], + ['whitespace-only body', 'blank.md', NONCE, ' \n\t\n'], + ['malformed nonce', 'valid.md', 'not-a-nonce', '### Assessment: 🟢 NO FINDINGS\n'], + ['forbidden wording', 'lgtm.md', NONCE, '### Assessment: 🟢 NO FINDINGS\n\nLGTM!\n'], + ['missing status line', 'prose.md', NONCE, 'looks fine to me\n'], + ])('refuses posting on %s', (_name, file, nonce, content) => { + const bodyFile = file.includes('/') ? file : join(directory, file); + if (content !== undefined) writeFileSync(bodyFile, content); + const commentsFile = stageComments('[]\n'); + expect(() => main(['finalize-body', bodyFile, nonce, commentsFile])).toThrow('exit:1'); + if (content !== undefined && content !== '') { + expect(readFileSync(bodyFile, 'utf8')).toBe(content); + } + }); + + it.each([ + ['missing comments argument', 'omit', 'requires the staged review comments file path'], + ['missing comments file', 'absent', 'missing or unreadable'], + ['non-JSON comments file', 'not json', 'not valid JSON'], + ['non-array comments file', '{"body": "x"}', 'must be a JSON array'], + ])('refuses posting on %s without touching the body', (_name, comments, reason) => { + const bodyFile = join(directory, 'review_body.md'); + const body = '### Assessment: 🟡 NEEDS ATTENTION\n'; + writeFileSync(bodyFile, body); + const argv = ['finalize-body', bodyFile, NONCE]; + if (comments === 'absent') argv.push(join(directory, 'nowhere.json')); + else if (comments !== 'omit') argv.push(stageComments(comments)); + expect(() => main(argv)).toThrow('exit:1'); + expect(errors.join('\n')).toContain(reason); + expect(readFileSync(bodyFile, 'utf8')).toBe(body); + }); + + it('refuses 🟢 NO FINDINGS over staged inline comments (runtime enforcement)', () => { + const bodyFile = join(directory, 'review_body.md'); + const body = '### Assessment: 🟢 NO FINDINGS\n'; + writeFileSync(bodyFile, body); + const commentsFile = stageComments('[{"path": "a.go", "line": 1, "body": "**[low] x**"}]\n'); + expect(() => main(['finalize-body', bodyFile, NONCE, commentsFile])).toThrow('exit:1'); + expect(errors.join('\n')).toContain( + '🟢 NO FINDINGS cannot be posted with 1 staged inline comment', + ); + expect(readFileSync(bodyFile, 'utf8')).toBe(body); + }); +}); + +describe('classify-run argument validation', () => { + it.each([ + ['short SHA', ['classify-run', 'abc', '100', NONCE]], + ['non-numeric baseline', ['classify-run', 'a'.repeat(40), '10x', NONCE]], + ['malformed nonce', ['classify-run', 'a'.repeat(40), '100', 'zz']], + ['unknown command', ['what-is-this']], + ['no command', []], + ])('exits nonzero on %s before touching stdin', (_name, argv) => { + expect(() => main(argv)).toThrow('exit:1'); + expect(stdout).toEqual([]); + }); +}); diff --git a/src/review-assessment/__tests__/review-assessment.test.ts b/src/review-assessment/__tests__/review-assessment.test.ts new file mode 100644 index 0000000..a49b975 --- /dev/null +++ b/src/review-assessment/__tests__/review-assessment.test.ts @@ -0,0 +1,497 @@ +// Copyright The Docker Agent Action authors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Unit tests for the review-assessment module. + * + * assessReview() is a tested MIRROR of the Decision Rules prose in + * review-pr/agents/pr-review.yaml — the model applies the rules, these tests + * pin the policy outcomes. The rest of the module is runtime logic bundled to + * dist/review-assessment.js and invoked by review-pr/action.yml: the + * finalize-body posting validator, the per-run attribution marker, and the + * fail-closed post-run classification (the shell harness in + * src/resolve-trigger-context/__tests__/workflow-security.test.ts executes + * the same code through the built CLI). + * + * The outcomes pinned are the ones that regressed in production + * (docker/gordon PRs #1798/#1803/#1808/#1809 false LGTMs, #1814 an approving + * zero-finding label over three surviving low findings): only completed-run + * headers may advance the last-reviewed SHA, and no header — the zero-findings + * one included — may ever present the bot as approving the PR. + */ +import { describe, expect, it } from 'vitest'; +import { findLastReviewedSha } from '../../incremental-review/incremental-review.js'; +import { + ASSESSMENT_MARKER, + type AssessmentInput, + assessReview, + classifyReviewBody, + classifyRunReviews, + finalizeReviewBody, + hasRunMarker, + INCOMPLETE_HEADER, + INCONCLUSIVE_HEADER, + isActionPostedReview, + isValidRunNonce, + NO_FINDINGS_LABEL, + type PostedReviewLike, + runMarker, + type SurvivingFinding, +} from '../review-assessment.js'; + +const high = (disposition: SurvivingFinding['disposition'] = 'inline'): SurvivingFinding => ({ + severity: 'high', + disposition, + verdict: 'CONFIRMED', +}); +const medium = (disposition: SurvivingFinding['disposition'] = 'inline'): SurvivingFinding => ({ + severity: 'medium', + disposition, + verdict: 'LIKELY', +}); +/** Unverified low finding: verification is skipped, no verdict. */ +const low = (): SurvivingFinding => ({ severity: 'low', disposition: 'summary' }); + +function complete(survivingFindings: SurvivingFinding[]): AssessmentInput { + return { reviewComplete: true, verificationConclusive: true, survivingFindings }; +} + +describe('assessReview', () => { + it('labels a complete, conclusive review with zero surviving findings 🟢 NO FINDINGS', () => { + const outcome = assessReview(complete([])); + expect(outcome).toEqual({ + kind: 'no-findings', + header: '### Assessment: 🟢 NO FINDINGS', + completedRun: true, + }); + }); + + it('never reports zero findings on a single surviving low finding (summary-only)', () => { + const outcome = assessReview(complete([low()])); + expect(outcome.kind).toBe('needs-attention'); + expect(outcome.header).toBe('### Assessment: 🟡 NEEDS ATTENTION'); + expect(outcome.header).not.toContain(NO_FINDINGS_LABEL); + expect(outcome.completedRun).toBe(true); + }); + + it('never reports zero findings when only summary-disposition findings survive', () => { + // The #1814 regression: assessment was driven by inline findings only, so + // three surviving low findings were silently dropped into a clean label. + const outcome = assessReview(complete([low(), low(), low()])); + expect(outcome.kind).toBe('needs-attention'); + }); + + it('labels surviving medium findings as needs-attention regardless of disposition', () => { + expect(assessReview(complete([medium('summary')])).kind).toBe('needs-attention'); + expect(assessReview(complete([medium()])).kind).toBe('needs-attention'); + }); + + it('labels any surviving verified high finding as critical', () => { + const outcome = assessReview(complete([low(), medium(), high()])); + expect(outcome.kind).toBe('critical'); + expect(outcome.header).toBe('### Assessment: 🔴 CRITICAL'); + }); + + it('reports incomplete reviews without an assessment marker at any finding count', () => { + for (const survivingFindings of [[], [low()], [high(), medium()]]) { + const outcome = assessReview({ + reviewComplete: false, + verificationConclusive: true, + survivingFindings, + }); + expect(outcome.kind).toBe('incomplete'); + expect(outcome.header).toBe(INCOMPLETE_HEADER); + expect(outcome.header).not.toContain(ASSESSMENT_MARKER); + expect(outcome.completedRun).toBe(false); + } + }); + + it('incompleteness overrides inconclusive verification', () => { + const outcome = assessReview({ + reviewComplete: false, + verificationConclusive: false, + survivingFindings: [], + }); + expect(outcome.kind).toBe('incomplete'); + }); + + it('reports inconclusive verification without an assessment marker or clean label', () => { + for (const survivingFindings of [[], [medium()], [high()]]) { + const outcome = assessReview({ + reviewComplete: true, + verificationConclusive: false, + survivingFindings, + }); + expect(outcome.kind).toBe('inconclusive'); + expect(outcome.header).toBe(INCONCLUSIVE_HEADER); + expect(outcome.header).not.toContain(ASSESSMENT_MARKER); + expect(outcome.completedRun).toBe(false); + } + }); + + it('never emits the NO FINDINGS label outside the zero-findings outcome', () => { + const inputs: AssessmentInput[] = [ + complete([low()]), + complete([medium()]), + complete([high()]), + { reviewComplete: false, verificationConclusive: true, survivingFindings: [] }, + { reviewComplete: true, verificationConclusive: false, survivingFindings: [] }, + ]; + for (const input of inputs) { + expect(assessReview(input).header).not.toContain(NO_FINDINGS_LABEL); + } + }); + + it('never emits approve or LGTM wording in ANY outcome, zero-findings included', () => { + // The bot only ever posts COMMENT reviews; no body header may present it + // as approving the PR — the legacy "🟢 APPROVE" label must never return. + const inputs: AssessmentInput[] = [ + complete([]), + complete([low()]), + complete([medium()]), + complete([high()]), + { reviewComplete: false, verificationConclusive: true, survivingFindings: [] }, + { reviewComplete: false, verificationConclusive: true, survivingFindings: [high()] }, + { reviewComplete: true, verificationConclusive: false, survivingFindings: [] }, + ]; + for (const input of inputs) { + const header = assessReview(input).header; + expect(header).not.toMatch(/approve|lgtm|no issues found/i); + } + }); +}); + +describe('incremental checkpoint cross-check', () => { + const SHA = 'a'.repeat(40); + + function botReview(body: string) { + return { + user: { login: 'docker-agent' }, + body, + commit_id: SHA, + submitted_at: '2026-01-01T10:00:00Z', + }; + } + + it.each([ + complete([]), + complete([low()]), + complete([high()]), + { reviewComplete: false, verificationConclusive: true, survivingFindings: [] }, + { reviewComplete: false, verificationConclusive: true, survivingFindings: [high()] }, + { reviewComplete: true, verificationConclusive: false, survivingFindings: [medium()] }, + ] as AssessmentInput[])('advances the checkpoint iff the outcome is a completed run (%j)', (input) => { + const outcome = assessReview(input); + const sha = findLastReviewedSha([botReview(outcome.header)]); + expect(sha).toBe(outcome.completedRun ? SHA : null); + }); +}); + +const NONCE = '0123456789abcdef0123456789abcdef'; +const MARKER = ``; + +describe('run marker primitives', () => { + it('accepts only 32 lowercase hex nonces', () => { + expect(isValidRunNonce(NONCE)).toBe(true); + for (const bad of ['', 'g'.repeat(32), NONCE.slice(1), `${NONCE}0`, NONCE.toUpperCase()]) { + expect(isValidRunNonce(bad), bad).toBe(false); + expect(() => runMarker(bad), bad).toThrow(/32 lowercase hex/); + } + }); + + it('builds the fixed-format marker and recognizes it with any nonce', () => { + expect(runMarker(NONCE)).toBe(MARKER); + expect(hasRunMarker(`### Assessment: 🟡 NEEDS ATTENTION\n\n${MARKER}`)).toBe(true); + expect(hasRunMarker(`x y`)).toBe(true); + // Malformed variants never match — attribution needs the exact format. + expect(hasRunMarker('')).toBe(false); + expect(hasRunMarker('')).toBe(false); + expect(hasRunMarker(null)).toBe(false); + }); +}); + +describe('isActionPostedReview', () => { + it('accepts the legacy docker-agent login variants without a marker', () => { + expect(isActionPostedReview('docker-agent', 'any body')).toBe(true); + expect(isActionPostedReview('docker-agent[bot]', 'any body')).toBe(true); + }); + + it('accepts marker-bearing reviews from [bot]-suffixed logins (default token)', () => { + expect(isActionPostedReview('github-actions[bot]', `body\n${MARKER}`)).toBe(true); + expect(isActionPostedReview('consumer-app[bot]', `body\n${MARKER}`)).toBe(true); + }); + + it('rejects marker-bearing reviews from human logins (forgeable marker)', () => { + // GitHub reserves the [bot] suffix for installed Apps; a PR author could + // paste a well-formed marker into their own review, so plain user logins + // must never qualify — a forged checkpoint would skip unreviewed commits. + expect(isActionPostedReview('mallory', `body\n${MARKER}`)).toBe(false); + expect(isActionPostedReview('github-actions[bot]', 'no marker')).toBe(false); + expect(isActionPostedReview(null, `body\n${MARKER}`)).toBe(false); + }); +}); + +describe('classifyReviewBody', () => { + it.each([ + '### Assessment: 🟢 NO FINDINGS', + '### Assessment: 🟡 NEEDS ATTENTION', + '### Assessment: 🔴 CRITICAL', + ])('classifies %s as completed', (line) => { + expect(classifyReviewBody(line)).toEqual({ kind: 'completed', assessment: line }); + }); + + it('keeps legitimate note text before the single status line supported', () => { + const body = [ + 'This review covers only the commits since `abc123def456`.', + '', + '### Assessment: 🟡 NEEDS ATTENTION', + '', + '#### Lower-confidence findings (not posted inline)', + '- [medium] file.go:42 — issue (confidence: weak 48/100)', + ].join('\n'); + expect(classifyReviewBody(body).kind).toBe('completed'); + }); + + it('maps the incomplete and inconclusive headers to their statuses', () => { + expect(classifyReviewBody(`${INCOMPLETE_HEADER}\nchunk 2: Drafter did not complete`)).toEqual({ + kind: 'incomplete', + }); + expect(classifyReviewBody('⚠️ **Review incomplete** — no review was posted.')).toEqual({ + kind: 'incomplete', + }); + expect(classifyReviewBody(`${INCONCLUSIVE_HEADER}\nUnverified findings below.`)).toEqual({ + kind: 'inconclusive', + }); + }); + + it('lets incompleteness outrank inconclusive verification (rule 4 combination)', () => { + expect( + classifyReviewBody(`${INCOMPLETE_HEADER}\n\n${INCONCLUSIVE_HEADER} for chunk 2.`), + ).toEqual({ kind: 'incomplete' }); + }); + + it.each([ + ['bodyless', ''], + ['whitespace-only', ' \n\t '], + ['unknown body', 'Some prose that never states an outcome.'], + ['unknown assessment label', '### Assessment: 🟣 MYSTERY'], + ['legacy approve label', '### Assessment: 🟢 APPROVE'], + ['assessment not on its own line', 'note ### Assessment: 🟡 NEEDS ATTENTION trailing'], + ['conflicting assessment + incomplete', `${INCOMPLETE_HEADER}\n### Assessment: 🟢 NO FINDINGS`], + [ + 'conflicting assessment + inconclusive', + `${INCONCLUSIVE_HEADER}\n### Assessment: 🔴 CRITICAL`, + ], + [ + 'multiple assessment lines', + '### Assessment: 🟡 NEEDS ATTENTION\n### Assessment: 🔴 CRITICAL', + ], + ['LGTM wording', '### Assessment: 🟢 NO FINDINGS\n\nLGTM!'], + ['lowercase lgtm wording', '### Assessment: 🟢 NO FINDINGS\n\nlgtm 🚀'], + ['APPROVE wording', '### Assessment: 🟡 NEEDS ATTENTION\n\nI APPROVE this change.'], + ['APPROVED wording', 'APPROVED\n### Assessment: 🟢 NO FINDINGS'], + ['no-issues wording', '🟢 **No issues found** — all good.'], + [ + 'NO FINDINGS over a findings section', + '### Assessment: 🟢 NO FINDINGS\n\n#### Low-severity findings (not verified, not posted inline)\n- [low] a.go:1 — x', + ], + [ + 'NO FINDINGS over a lower-confidence section', + '### Assessment: 🟢 NO FINDINGS\n\n#### Lower-confidence findings (not posted inline)\n- [medium] a.go:1 — x', + ], + [ + 'NO FINDINGS over an inline findings section', + '### Assessment: 🟢 NO FINDINGS\n\n### Findings\n**[high] a.go:1 — x**', + ], + ])('fails closed on %s', (_name, body) => { + expect(classifyReviewBody(body).kind).toBe('invalid'); + }); + + it('does not refuse prose words that merely contain approve', () => { + const body = + '### Assessment: 🟡 NEEDS ATTENTION\n\nThe approveTransfer() call skips validation.'; + expect(classifyReviewBody(body).kind).toBe('completed'); + }); +}); + +describe('finalizeReviewBody', () => { + it('appends the run marker exactly once (idempotent for the same nonce)', () => { + const once = finalizeReviewBody('### Assessment: 🟢 NO FINDINGS\n', NONCE, 0); + expect(once).toBe(`### Assessment: 🟢 NO FINDINGS\n\n${MARKER}\n`); + expect(finalizeReviewBody(once, NONCE, 0)).toBe(once); + }); + + it('refuses invalid bodies with the classification reason', () => { + expect(() => finalizeReviewBody('LGTM!', NONCE, 0)).toThrow(/refusing to post review body/); + expect(() => finalizeReviewBody('no status here', NONCE, 0)).toThrow( + /no recognized status line/, + ); + }); + + it('refuses a body carrying a marker from a different run', () => { + const stale = `### Assessment: 🟢 NO FINDINGS\n\n`; + expect(() => finalizeReviewBody(stale, NONCE, 0)).toThrow(/different run/); + }); + + it('refuses 🟢 NO FINDINGS over staged inline comments', () => { + // The zero-findings label asserts zero surviving findings of every + // severity — a staged inline comment contradicts it at posting time. + expect(() => finalizeReviewBody('### Assessment: 🟢 NO FINDINGS\n', NONCE, 1)).toThrow( + /🟢 NO FINDINGS cannot be posted with 1 staged inline comment/, + ); + expect(() => finalizeReviewBody('### Assessment: 🟢 NO FINDINGS\n', NONCE, 3)).toThrow( + /3 staged inline comment/, + ); + }); + + it('refuses malformed comment counts instead of guessing', () => { + for (const count of [-1, 0.5, Number.NaN]) { + expect(() => + finalizeReviewBody('### Assessment: 🟡 NEEDS ATTENTION\n', NONCE, count), + ).toThrow(/non-negative integer/); + } + }); + + it('accepts staged inline comments for every other recognized outcome', () => { + for (const body of [ + '### Assessment: 🟡 NEEDS ATTENTION\n', + '### Assessment: 🔴 CRITICAL\n', + '### ⚠️ Review incomplete\nchunk 2: Drafter did not complete\n', + '### ⚠️ Verification inconclusive\nUnverified findings below.\n', + ]) { + expect(finalizeReviewBody(body, NONCE, 2), body).toContain(MARKER); + } + }); +}); + +describe('classifyRunReviews', () => { + const SHA = 'a'.repeat(40); + const OTHER_SHA = 'b'.repeat(40); + const BASELINE = 100; + const opts = { sha: SHA, baselineId: BASELINE, nonce: NONCE }; + + function posted(overrides: Partial = {}): PostedReviewLike { + return { + id: 101, + user: { login: 'github-actions[bot]' }, + body: `### Assessment: 🟡 NEEDS ATTENTION\n\n${MARKER}\n`, + commit_id: SHA, + state: 'COMMENTED', + ...overrides, + }; + } + + it('classifies exactly one valid COMMENTED marker-bearing review as completed', () => { + const result = classifyRunReviews([posted()], opts); + expect(result.status).toBe('completed'); + expect(result.reviewId).toBe(101); + }); + + it('attributes by exact marker regardless of the posting login', () => { + for (const login of ['docker-agent', 'docker-agent[bot]', 'consumer-machine-user', null]) { + expect(classifyRunReviews([posted({ user: { login } })], opts).status, String(login)).toBe( + 'completed', + ); + } + }); + + it('maps agent-posted incomplete and inconclusive bodies to their statuses', () => { + const incomplete = posted({ + body: `${INCOMPLETE_HEADER}\nchunk 2: Drafter did not complete\n\n${MARKER}\n`, + }); + expect(classifyRunReviews([incomplete], opts).status).toBe('incomplete'); + const inconclusive = posted({ + body: `${INCONCLUSIVE_HEADER}\nUnverified findings below.\n\n${MARKER}\n`, + }); + expect(classifyRunReviews([inconclusive], opts).status).toBe('inconclusive'); + }); + + it('reports none when nothing carries the marker (unrelated same-SHA human reviews ignored)', () => { + const result = classifyRunReviews( + [ + { id: 90, user: { login: 'docker-agent' }, body: 'old review', commit_id: SHA }, + { id: 105, user: { login: 'human-reviewer' }, body: 'nice', commit_id: SHA }, + ], + opts, + ); + expect(result.status).toBe('none'); + }); + + it.each([ + ['stale marker at/below the baseline', [posted({ id: BASELINE })]], + ['marker on a different SHA', [posted({ commit_id: OTHER_SHA })]], + ['duplicate exact-marker reviews', [posted(), posted({ id: 102 })]], + ['non-numeric review ID', [posted({ id: null })]], + ['PENDING state', [posted({ state: 'PENDING' })]], + ['APPROVED state', [posted({ state: 'APPROVED' })]], + ['CHANGES_REQUESTED state', [posted({ state: 'CHANGES_REQUESTED' })]], + ['missing state', [posted({ state: undefined })]], + [ + 'conflicting body', + [posted({ body: `${INCOMPLETE_HEADER}\n### Assessment: 🟢 NO FINDINGS\n${MARKER}` })], + ], + ['unknown body', [posted({ body: `mystery\n${MARKER}` })]], + ['LGTM body', [posted({ body: `### Assessment: 🟢 NO FINDINGS\nLGTM!\n${MARKER}` })]], + ['legacy approve label', [posted({ body: `### Assessment: 🟢 APPROVE\n${MARKER}` })]], + [ + 'fresh same-SHA bot review without a marker', + [ + { + id: 101, + user: { login: 'github-actions[bot]' }, + body: 'no marker', + commit_id: SHA, + state: 'COMMENTED', + }, + ], + ], + [ + 'fresh same-SHA docker-agent review without a marker', + [ + { + id: 101, + user: { login: 'docker-agent' }, + body: '### Assessment: 🟢 NO FINDINGS', + commit_id: SHA, + state: 'COMMENTED', + }, + ], + ], + ] as [string, PostedReviewLike[]][])('fails closed as unverified on %s', (_name, reviews) => { + expect(classifyRunReviews(reviews, opts).status).toBe('unverified'); + }); + + it('fails closed on malformed inputs instead of guessing', () => { + expect(classifyRunReviews([posted()], { ...opts, sha: 'nope' }).status).toBe('unverified'); + expect(classifyRunReviews([posted()], { ...opts, baselineId: -1 }).status).toBe('unverified'); + expect(classifyRunReviews([posted()], { ...opts, baselineId: 0.5 }).status).toBe('unverified'); + expect(classifyRunReviews([posted()], { ...opts, nonce: 'short' }).status).toBe('unverified'); + }); + + it('reports a prior incomplete notice pinned to the same SHA by an action identity', () => { + const notice = (login: string, body: string, commitId = SHA): PostedReviewLike => ({ + id: 60, + user: { login }, + body, + commit_id: commitId, + state: 'COMMENTED', + }); + const plain = '⚠️ **Review incomplete** — The review agent finished without posting a review.'; + const marked = `${plain}\n\n`; + expect(classifyRunReviews([notice('docker-agent', plain)], opts).priorIncompleteNotice).toBe( + true, + ); + expect( + classifyRunReviews([notice('docker-agent[bot]', plain)], opts).priorIncompleteNotice, + ).toBe(true); + // Custom-token notices are recognized by their (older) run marker. + expect( + classifyRunReviews([notice('github-actions[bot]', marked)], opts).priorIncompleteNotice, + ).toBe(true); + // Human same-worded reviews and other-SHA notices never dedup. + expect(classifyRunReviews([notice('human', plain)], opts).priorIncompleteNotice).toBe(false); + expect( + classifyRunReviews([notice('docker-agent', plain, OTHER_SHA)], opts).priorIncompleteNotice, + ).toBe(false); + }); +}); diff --git a/src/review-assessment/index.ts b/src/review-assessment/index.ts new file mode 100644 index 0000000..343ef40 --- /dev/null +++ b/src/review-assessment/index.ts @@ -0,0 +1,134 @@ +// Copyright The Docker Agent Action authors +// SPDX-License-Identifier: Apache-2.0 + +/** + * review-assessment CLI entrypoint (bundled to dist/review-assessment.js and + * staged at /tmp/review-assessment.js for the agent's posting command). + * + * Usage: + * node dist/review-assessment.js new-run-nonce + * Print a cryptographically random 32-hex per-run attribution nonce. + * Generated in a trusted action step, never from user input. + * + * node dist/review-assessment.js finalize-body + * Validate the agent-authored review body in against the + * posting policy (exactly one recognized status line, no APPROVE/LGTM/ + * "No issues found" wording, no 🟢 NO FINDINGS over findings sections) + * plus the staged inline comments in — which must parse + * as a JSON array, empty for a 🟢 NO FINDINGS body — and mechanically + * append this run's hidden attribution marker. Writes the finalized + * body back to . Nonzero exit refuses posting. + * + * node dist/review-assessment.js classify-run + * Read the PR's reviews JSON (array) from stdin and print the status of + * the review THIS run posted: completed | incomplete | inconclusive | + * none | unverified, one `status=` line plus a + * `prior-incomplete-notice=` line for the fallback dedup guard. + * The reason is logged to stderr. Nonzero exit means the classification + * itself could not run (bad arguments or unparseable input) — callers + * must treat that as unverified. + */ +import { randomBytes } from 'node:crypto'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { + classifyRunReviews, + finalizeReviewBody, + isValidRunNonce, + type PostedReviewLike, +} from './review-assessment.js'; + +export function newRunNonce(): string { + return randomBytes(16).toString('hex'); +} + +function fail(message: string): never { + console.error(`review-assessment: ${message}`); + process.exit(1); +} + +function requireNonce(nonce: string | undefined): string { + if (!nonce || !isValidRunNonce(nonce)) { + fail('nonce must be exactly 32 lowercase hex characters'); + } + return nonce; +} + +export function main(argv: string[]): void { + const [command, ...args] = argv; + + if (command === 'new-run-nonce') { + process.stdout.write(`${newRunNonce()}\n`); + return; + } + + if (command === 'finalize-body') { + const [bodyFile, nonce, commentsFile] = args; + if (!bodyFile) fail('finalize-body requires a body file path'); + const validNonce = requireNonce(nonce); + if (!commentsFile) fail('finalize-body requires the staged review comments file path'); + let body: string; + try { + body = readFileSync(bodyFile, 'utf8'); + } catch { + fail(`review body file ${bodyFile} is missing or unreadable`); + } + if (body.trim() === '') { + fail(`review body file ${bodyFile} is empty — write the computed outcome first`); + } + let commentsRaw: string; + try { + commentsRaw = readFileSync(commentsFile, 'utf8'); + } catch { + fail(`review comments file ${commentsFile} is missing or unreadable`); + } + let comments: unknown; + try { + comments = JSON.parse(commentsRaw); + } catch { + fail(`review comments file ${commentsFile} is not valid JSON`); + } + if (!Array.isArray(comments)) { + fail(`review comments file ${commentsFile} must be a JSON array of inline comments`); + } + let finalized: string; + try { + finalized = finalizeReviewBody(body, validNonce, comments.length); + } catch (error: unknown) { + fail(error instanceof Error ? error.message : String(error)); + } + writeFileSync(bodyFile, finalized); + console.error( + `review-assessment: body validated against ${comments.length} staged inline comment(s), run marker appended to ${bodyFile}`, + ); + return; + } + + if (command === 'classify-run') { + const [sha, baseline, nonce] = args; + if (!sha || !/^[0-9a-f]{40}$/i.test(sha)) fail('classify-run requires a 40-hex SHA'); + if (!baseline || !/^[0-9]+$/.test(baseline)) fail('classify-run requires a numeric baseline'); + const validNonce = requireNonce(nonce); + let reviews: unknown; + try { + reviews = JSON.parse(readFileSync(0, 'utf8')); + } catch { + fail('stdin is not valid JSON'); + } + if (!Array.isArray(reviews)) fail('stdin must be a JSON array of reviews'); + const classification = classifyRunReviews(reviews as PostedReviewLike[], { + sha, + baselineId: Number.parseInt(baseline, 10), + nonce: validNonce, + }); + console.error(`review-assessment: ${classification.reason}`); + process.stdout.write(`status=${classification.status}\n`); + process.stdout.write(`prior-incomplete-notice=${classification.priorIncompleteNotice}\n`); + return; + } + + fail(`unknown command ${JSON.stringify(command ?? '')}`); +} + +if (process.argv[1]?.endsWith('review-assessment.js') && !process.env.VITEST) { + main(process.argv.slice(2)); +} diff --git a/src/review-assessment/review-assessment.ts b/src/review-assessment/review-assessment.ts new file mode 100644 index 0000000..747c18c --- /dev/null +++ b/src/review-assessment/review-assessment.ts @@ -0,0 +1,479 @@ +// Copyright The Docker Agent Action authors +// SPDX-License-Identifier: Apache-2.0 + +/** + * review-assessment — Decision Rules policy plus the trusted runtime helpers + * behind the PR review posting path. The assessReview() half MIRRORS the + * rules prose in `review-pr/agents/pr-review.yaml` (same contract as + * src/score-confidence for the Confidence Scoring section): change one, + * change both — the unit tests pin every outcome. The rest of the module IS + * runtime code, bundled to dist/review-assessment.js (see index.ts) and + * invoked by review-pr/action.yml: + * - finalize-body validates the agent-authored review body before posting + * (exactly one status line, no approve/LGTM wording, no 🟢 NO FINDINGS + * over surviving findings or staged inline comments) and mechanically + * appends this run's hidden attribution marker; + * - classify-run classifies what a run actually posted from GitHub API + * state (exact run marker + selected SHA + ID above the pre-run + * baseline), fail-closed on anything ambiguous; + * - the run-marker predicates are shared with src/incremental-review + * (checkpoint recognition) and src/rate-limit (review counting). + * + * The model still applies the Decision Rules from the prompt — no code + * recomputes the full assessment — but a body whose status line violates them + * is refused at posting time, and a run whose posted review cannot be + * verified is never presented as completed. + * + * Terminology: a finding SURVIVES when it is in scope and was not dismissed + * or dropped — i.e. it will be surfaced anywhere in the posted review: as an + * inline comment, in the lower-confidence summary, in the medium-severity + * floor list, or in the unverified low-severity list. Out-of-scope, dropped + * (negligible-band low), DISMISSED, and audit-only (dismissed security) + * findings do not survive. + * + * Fail-closed invariants (regressions here caused real false approvals — + * docker/gordon PRs #1798/#1803/#1808/#1809/#1814): + * - The bot NEVER approves: the GitHub event is always COMMENT and no + * outcome header may carry approve wording. The zero-findings label is + * the neutral "🟢 NO FINDINGS", never "🟢 APPROVE" or an LGTM. + * - An incomplete review (any drafter chunk errored, refused, or was + * truncated) never carries the "### Assessment:" completed-run marker, + * at ANY finding count. + * - Inconclusive verification (malformed/unpaired/refused verifier batch) + * likewise never carries the marker. + * - "🟢 NO FINDINGS" requires a complete review, conclusive verification, + * and exactly zero surviving findings of EVERY severity — a single + * surviving low finding forces a needs-attention assessment. + */ + +export type Severity = 'high' | 'medium' | 'low'; + +/** Where a surviving finding is surfaced in the posted review. */ +export type Disposition = 'inline' | 'summary'; + +export interface SurvivingFinding { + severity: Severity; + disposition: Disposition; + /** + * Verifier verdict for verified findings. Unverified low-severity findings + * (verification is skipped for them) carry none — they still survive. + */ + verdict?: 'CONFIRMED' | 'LIKELY'; +} + +export interface AssessmentInput { + /** Merged review_complete across every drafter delegation (fail-closed). */ + reviewComplete: boolean; + /** + * False when the verifier batch was malformed, refused, or failed pairing. + * True when verification succeeded or was not required (nothing to verify). + */ + verificationConclusive: boolean; + /** Findings that survive scope filtering, dismissal, and dropping. */ + survivingFindings: SurvivingFinding[]; +} + +/** Completed-run marker the incremental reviewer checkpoints on. */ +export const ASSESSMENT_MARKER = '### Assessment:'; +/** Body header for a review with unreviewed chunks — never a checkpoint. */ +export const INCOMPLETE_HEADER = '### ⚠️ Review incomplete'; +/** Body header for unverified surfaced findings — never a checkpoint. */ +export const INCONCLUSIVE_HEADER = '### ⚠️ Verification inconclusive'; + +export const CRITICAL_LABEL = '🔴 CRITICAL'; +export const NEEDS_ATTENTION_LABEL = '🟡 NEEDS ATTENTION'; +/** + * Neutral zero-findings completion label. Deliberately NOT an approval: + * reviews posted before the label was neutralized carry the legacy + * "🟢 APPROVE" — recognized as completed runs by src/incremental-review via + * the "### Assessment:" marker, but never emitted again. + */ +export const NO_FINDINGS_LABEL = '🟢 NO FINDINGS'; + +/** + * Hidden per-run attribution marker. A trusted pre-run step generates the + * 32-hex nonce (Node crypto); the rendered posting command and every + * fallback notice embed the full marker, so post-run classification can + * attribute reviews to exactly one run by content — independent of the + * posting login, which varies with the github-token input (docker-agent PAT, + * docker-agent[bot]/github-actions[bot] app tokens, or a consumer identity). + */ +export const RUN_MARKER_PREFIX = ''; +export const RUN_NONCE_PATTERN = /^[0-9a-f]{32}$/; +// Well-formed marker with ANY nonce — recognition across runs (incremental +// checkpoint, rate counting, notice dedup), where the per-run nonce differs. +const ANY_RUN_MARKER = //; + +export function isValidRunNonce(nonce: string): boolean { + return RUN_NONCE_PATTERN.test(nonce); +} + +/** Build the exact marker for one run. Throws on a malformed nonce. */ +export function runMarker(nonce: string): string { + if (!isValidRunNonce(nonce)) { + throw new Error('run nonce must be exactly 32 lowercase hex characters'); + } + return `${RUN_MARKER_PREFIX}${nonce}${RUN_MARKER_SUFFIX}`; +} + +/** True when the body carries a well-formed run marker (any nonce). */ +export function hasRunMarker(body: string | null | undefined): boolean { + return ANY_RUN_MARKER.test(body ?? ''); +} + +// GitHub presents the action's posting identity as "docker-agent" (machine +// user PAT — what setup-credentials stages) or "docker-agent[bot]" (GitHub +// App installation token). +export function matchesBotLogin(login: string | null | undefined, botLogin: string): boolean { + return login === botLogin || login === `${botLogin}[bot]`; +} + +/** + * Whether a review is recognizable as posted by this action across runs: + * the legacy docker-agent login variants, or a run-marker-bearing body from + * a `[bot]`-suffixed login (default github.token posts as + * "github-actions[bot]"; App tokens as "[bot]"). + * + * The `[bot]` requirement is deliberate: GitHub reserves that suffix for + * installed Apps, so a human PR author cannot mint a marker-bearing review + * that pins the incremental checkpoint past unreviewed commits or inflates + * the rate count. Consumers posting with a plain PAT identity fall back to + * full reviews — safe, just not incremental. + */ +export function isActionPostedReview( + login: string | null | undefined, + body: string | null | undefined, + botLogin = 'docker-agent', +): boolean { + if (matchesBotLogin(login, botLogin)) return true; + return typeof login === 'string' && login.endsWith('[bot]') && hasRunMarker(body); +} + +export type AssessmentKind = + | 'incomplete' + | 'inconclusive' + | 'critical' + | 'needs-attention' + | 'no-findings'; + +export interface AssessmentOutcome { + kind: AssessmentKind; + /** First line of the posted review body. */ + header: string; + /** + * True only when the body may carry the "### Assessment:" completed-run + * marker, i.e. the run may advance the incremental review checkpoint. + */ + completedRun: boolean; +} + +/** + * Compute the review assessment. The GitHub review event is always COMMENT + * regardless of the outcome — the assessment is the honest label inside the + * body, never an APPROVE/REQUEST_CHANGES event, and no header ever presents + * the bot as approving the PR. + */ +export function assessReview(input: AssessmentInput): AssessmentOutcome { + if (!input.reviewComplete) { + return { kind: 'incomplete', header: INCOMPLETE_HEADER, completedRun: false }; + } + if (!input.verificationConclusive) { + return { kind: 'inconclusive', header: INCONCLUSIVE_HEADER, completedRun: false }; + } + const critical = input.survivingFindings.some( + (finding) => + finding.severity === 'high' && + (finding.verdict === 'CONFIRMED' || finding.verdict === 'LIKELY'), + ); + if (critical) { + return { + kind: 'critical', + header: `${ASSESSMENT_MARKER} ${CRITICAL_LABEL}`, + completedRun: true, + }; + } + if (input.survivingFindings.length > 0) { + return { + kind: 'needs-attention', + header: `${ASSESSMENT_MARKER} ${NEEDS_ATTENTION_LABEL}`, + completedRun: true, + }; + } + return { + kind: 'no-findings', + header: `${ASSESSMENT_MARKER} ${NO_FINDINGS_LABEL}`, + completedRun: true, + }; +} + +// --------------------------------------------------------------------------- +// Review body classification (finalize-body validation + posted-run status) +// --------------------------------------------------------------------------- + +/** The only assessment lines a completed review body may carry. */ +export const ALLOWED_ASSESSMENT_LINES = [ + `${ASSESSMENT_MARKER} ${NO_FINDINGS_LABEL}`, + `${ASSESSMENT_MARKER} ${NEEDS_ATTENTION_LABEL}`, + `${ASSESSMENT_MARKER} ${CRITICAL_LABEL}`, +] as const; + +// Bold no-post fallback form (review-pr/action.yml's incomplete notice) — +// recognized alongside the agent-posted INCOMPLETE_HEADER. +const INCOMPLETE_NOTICE_PREFIX = '⚠️ **Review incomplete**'; + +// The bot never approves. Reviews are always COMMENT events, and no posted +// body may present the run as an approval — these exact strings are the +// wording pr-review.yaml already prohibits, matched as standalone words so +// prose like "approveTransfer" in a finding is not refused. +const FORBIDDEN_WORDING: { pattern: RegExp; label: string }[] = [ + { pattern: /🟢 APPROVE/, label: '"🟢 APPROVE"' }, + { pattern: /\bAPPROVED?\b/, label: '"APPROVE"' }, + { pattern: /\bLGTM\b/i, label: '"LGTM"' }, + { pattern: /no issues found/i, label: '"No issues found"' }, +]; + +// Findings sections that contradict a 🟢 NO FINDINGS assessment: the label is +// reserved for zero surviving findings of every severity, so a body carrying +// it plus any findings list is refused. +const FINDINGS_SECTION_MARKERS = [ + '# Findings', + '# Lower-confidence findings', + '# Low-severity findings', + '# Dismissed security findings', + 'Findings so far:', +]; + +export type BodyStatus = + | { kind: 'completed'; assessment: string } + | { kind: 'incomplete' } + | { kind: 'inconclusive' } + | { kind: 'invalid'; reason: string }; + +/** + * Classify a review body against the posting policy. Fail-closed: anything + * that is not exactly one recognized outcome is invalid. Prose before the + * single status line (e.g. the incremental-review coverage note) is fine. + */ +export function classifyReviewBody(body: string | null | undefined): BodyStatus { + const text = body ?? ''; + if (text.trim() === '') return { kind: 'invalid', reason: 'body is empty' }; + + for (const { pattern, label } of FORBIDDEN_WORDING) { + if (pattern.test(text)) { + return { kind: 'invalid', reason: `body contains forbidden approval wording ${label}` }; + } + } + + const assessmentMentions = text.split(ASSESSMENT_MARKER).length - 1; + const incomplete = text.includes(INCOMPLETE_HEADER) || text.includes(INCOMPLETE_NOTICE_PREFIX); + const inconclusive = text.includes(INCONCLUSIVE_HEADER); + + if (incomplete || inconclusive) { + if (assessmentMentions > 0) { + return { + kind: 'invalid', + reason: 'body mixes an assessment line with an incomplete/inconclusive marker', + }; + } + // Rule 4: incompleteness outranks inconclusive verification, so a body + // carrying both is an incomplete review that also notes the inconclusive + // verification — not a conflict. + return incomplete ? { kind: 'incomplete' } : { kind: 'inconclusive' }; + } + + if (assessmentMentions === 0) { + return { kind: 'invalid', reason: 'body carries no recognized status line' }; + } + if (assessmentMentions > 1) { + return { kind: 'invalid', reason: 'body carries more than one assessment line' }; + } + const lines = text.split('\n').map((line) => line.trimEnd()); + const assessment = ALLOWED_ASSESSMENT_LINES.find((allowed) => lines.includes(allowed)); + if (!assessment) { + return { kind: 'invalid', reason: 'assessment line is not one of the allowed labels' }; + } + if ( + assessment === `${ASSESSMENT_MARKER} ${NO_FINDINGS_LABEL}` && + FINDINGS_SECTION_MARKERS.some((marker) => text.includes(marker)) + ) { + return { + kind: 'invalid', + reason: '🟢 NO FINDINGS cannot be combined with findings sections in the same body', + }; + } + return { kind: 'completed', assessment }; +} + +/** + * Validate an agent-authored body against the staged inline-comment count + * and mechanically append this run's marker. Returns the finalized body; + * throws with the refusal reason otherwise. The count is the trusted length + * of /tmp/review_comments.json: 🟢 NO FINDINGS asserts zero surviving + * findings, so it is refused over ANY staged inline comment; every other + * recognized outcome may carry comments. Idempotent for the same nonce (a + * retried posting command must not double-append); any OTHER run marker in + * the body is refused — the model must never copy a stale marker from logs + * or previous reviews. + */ +export function finalizeReviewBody(body: string, nonce: string, commentCount: number): string { + const marker = runMarker(nonce); + if (!Number.isInteger(commentCount) || commentCount < 0) { + throw new Error('refusing to post review body: comment count must be a non-negative integer'); + } + const status = classifyReviewBody(body); + if (status.kind === 'invalid') { + throw new Error(`refusing to post review body: ${status.reason}`); + } + if ( + status.kind === 'completed' && + status.assessment === `${ASSESSMENT_MARKER} ${NO_FINDINGS_LABEL}` && + commentCount > 0 + ) { + throw new Error( + `refusing to post review body: 🟢 NO FINDINGS cannot be posted with ${commentCount} staged inline comment(s)`, + ); + } + const foreign = body.replaceAll(marker, ''); + if (hasRunMarker(foreign)) { + throw new Error('refusing to post review body: it carries a run marker from a different run'); + } + if (body.includes(marker)) return body; + return `${body.replace(/\s+$/, '')}\n\n${marker}\n`; +} + +// --------------------------------------------------------------------------- +// Post-run classification (what did THIS run post?) +// --------------------------------------------------------------------------- + +/** Review shape as GET /pulls/{n}/reviews returns it (fields we read). */ +export interface PostedReviewLike { + id?: number | null; + user?: { login?: string | null } | null; + body?: string | null; + commit_id?: string | null; + state?: string | null; +} + +export type RunReviewStatus = 'completed' | 'incomplete' | 'inconclusive' | 'none' | 'unverified'; + +export interface RunClassification { + status: RunReviewStatus; + reason: string; + /** ID of the single attributed review, when status is not none/unverified. */ + reviewId: number | null; + /** + * True when a prior action run already pinned an incomplete-review notice + * to the same SHA — the caller's dedup guard for the no-post fallback. + */ + priorIncompleteNotice: boolean; +} + +const SHA40 = /^[0-9a-f]{40}$/i; + +/** + * Classify the review THIS run posted from GitHub API state. Attribution is + * exact: the run's unguessable marker, on the selected immutable SHA, with a + * review ID above the pre-run baseline. Fail closed (`unverified`) on + * anything ambiguous — duplicated markers, markers off the selected SHA or + * at/below the baseline (stale/copied), non-COMMENTED state, and fresh + * same-SHA bot-identity reviews that lack the marker (a bypassed template + * cannot be told apart from another integration's post). + */ +export function classifyRunReviews( + reviews: PostedReviewLike[], + opts: { sha: string; baselineId: number; nonce: string }, +): RunClassification { + if (!SHA40.test(opts.sha)) { + return { + status: 'unverified', + reason: 'selected head SHA is not a 40-hex commit', + reviewId: null, + priorIncompleteNotice: false, + }; + } + if (!Number.isInteger(opts.baselineId) || opts.baselineId < 0) { + return { + status: 'unverified', + reason: 'pre-run review baseline is not a review ID', + reviewId: null, + priorIncompleteNotice: false, + }; + } + let marker: string; + try { + marker = runMarker(opts.nonce); + } catch { + return { + status: 'unverified', + reason: 'run nonce is malformed', + reviewId: null, + priorIncompleteNotice: false, + }; + } + + const sameSha = (review: PostedReviewLike): boolean => + (review.commit_id ?? '').toLowerCase() === opts.sha.toLowerCase(); + const priorIncompleteNotice = reviews.some( + (review) => + sameSha(review) && + (review.body ?? '').startsWith(INCOMPLETE_NOTICE_PREFIX) && + isActionPostedReview(review.user?.login, review.body), + ); + const result = (status: RunReviewStatus, reason: string, reviewId: number | null = null) => ({ + status, + reason, + reviewId, + priorIncompleteNotice, + }); + + const marked = reviews.filter((review) => (review.body ?? '').includes(marker)); + if (marked.length === 0) { + // No marker anywhere. A fresh same-SHA review from a bot identity is + // unattributable (template bypassed? another integration?) — fail closed + // instead of posting a duplicate no-post notice next to it. Fresh human + // reviews on the same SHA are unrelated and never count. + const unmarkedBot = reviews.some( + (review) => + sameSha(review) && + typeof review.id === 'number' && + review.id > opts.baselineId && + (matchesBotLogin(review.user?.login, 'docker-agent') || + (review.user?.login ?? '').endsWith('[bot]')), + ); + if (unmarkedBot) { + return result( + 'unverified', + 'a fresh same-SHA bot review carries no run marker — cannot attribute it to this run', + ); + } + return result('none', 'no review carries this run\u2019s marker'); + } + if (marked.length > 1) { + return result('unverified', 'multiple reviews carry this run\u2019s marker'); + } + + const review = marked[0]; + if (typeof review.id !== 'number' || review.id <= opts.baselineId) { + return result('unverified', 'marker-bearing review predates the pre-run baseline'); + } + if (!sameSha(review)) { + return result('unverified', 'marker-bearing review is not on the selected SHA'); + } + if ((review.state ?? '') !== 'COMMENTED') { + return result( + 'unverified', + `marker-bearing review has state ${JSON.stringify(review.state ?? '')}, expected COMMENTED`, + review.id, + ); + } + const status = classifyReviewBody(review.body); + if (status.kind === 'invalid') { + return result( + 'unverified', + `marker-bearing review body is invalid: ${status.reason}`, + review.id, + ); + } + return result(status.kind, `review ${review.id} classified as ${status.kind}`, review.id); +} diff --git a/tsup.config.ts b/tsup.config.ts index 33fd9ab..804a380 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -37,6 +37,7 @@ const entry = { 'prepare-review': src('prepare-review'), 'rate-limit': src('rate-limit'), 'resolve-trigger-context': src('resolve-trigger-context'), + 'review-assessment': src('review-assessment'), 'score-confidence': src('score-confidence'), 'score-risk': src('score-risk'), security: src('security'),