diff --git a/agents/test-repair/Dockerfile b/agents/test-repair/Dockerfile new file mode 100644 index 0000000000..a03d3ca373 --- /dev/null +++ b/agents/test-repair/Dockerfile @@ -0,0 +1,67 @@ +FROM python:3.12 AS builder + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV UV_PROJECT_ENVIRONMENT=/opt/venv + +WORKDIR /app + +# Install external deps without building workspace members. +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ + --mount=type=bind,source=uv.lock,target=uv.lock \ + --mount=type=bind,source=VERSION,target=VERSION \ + uv sync --frozen --no-dev --no-install-workspace --package hackbot-agent-test-repair + +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,target=/app,rw \ + uv sync --locked --no-dev --no-editable --package hackbot-agent-test-repair + +FROM python:3.12 AS base + +COPY --from=builder /opt/venv /opt/venv +WORKDIR /app + +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PATH="/opt/venv/bin:$PATH" + +FROM base AS agent + +# Xvfb plus Firefox's runtime libraries and fonts, so the agent can actually run +# the failing test locally. `mach bootstrap` installs the *build* toolchain but +# runs unprivileged here, so it cannot install these system packages itself. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + xvfb x11-utils \ + fonts-liberation fonts-dejavu-core \ + libgtk-3-0t64 libdbus-glib-1-2 libx11-xcb1 libxtst6 libxt6 libpci3 \ + libasound2t64 libnss3 libnspr4 libgbm1 libdrm2 libxcomposite1 \ + libxdamage1 libxfixes3 libxrandr2 libxkbcommon0 libpango-1.0-0 \ + libcairo2 libatk1.0-0t64 libatk-bridge2.0-0t64 libcups2t64 \ + libatspi2.0-0t64 \ + zip unzip bzip2 \ + && rm -rf /var/lib/apt/lists/* + +# hackbot.toml lives at the agent root (not inside the package), so copy it into +# the working dir; the runtime discovers it there (cwd) at startup. +COPY agents/test-repair/hackbot.toml /app/hackbot.toml +COPY agents/test-repair/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +RUN useradd --create-home --shell /bin/bash agent \ + && mkdir -p /workspace \ + && chown agent:agent /workspace \ + # Xvfb runs unprivileged and cannot create its own socket directory. + && mkdir -p /tmp/.X11-unix \ + && chmod 1777 /tmp/.X11-unix + +# `mach bootstrap` installs the toolchain here at runtime; put it on PATH so the +# agent's own `./mach build` (and the build_firefox tool) find rustc/clang. +ENV PATH="/home/agent/.cargo/bin:/home/agent/.mozbuild/clang/bin:${PATH}" +ENV DISPLAY=:99 + +USER agent + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +CMD ["python", "-m", "hackbot_agents.test_repair"] diff --git a/agents/test-repair/README.md b/agents/test-repair/README.md new file mode 100644 index 0000000000..e11a3ccf52 --- /dev/null +++ b/agents/test-repair/README.md @@ -0,0 +1,74 @@ +# Test Repair Agent + +Two-stage Claude agent that finds the commit which regressed a failing Firefox CI +test and proposes a fix. Agent logic in `hackbot_agents/test_repair/`. + +The pulse listener only forwards failures that already passed its regression and +flakiness filters, so a regression is the prior, but the agent still reports the +classification it reaches. Its only input is a Taskcluster task id. + +Run the Docker command below from the repo root, with secrets in a local `.env` +(`ANTHROPIC_API_KEY`; `BUGZILLA_API_KEY` is optional). + +## Deterministic prep + +Before Claude is invoked, `resolve.py` turns the task id into everything the +investigation needs (no log parsing): + +1. Project + hg revision from the Taskcluster task. +2. The failing test groups, via mozci. +3. The revision at which the failing tests were last green, by walking mozci push + ancestors. Restricted to the failing tests on the failing platform, since a + manifest can be green on another platform or for its other tests. +4. The git range that landed since then, from the hg pushlog + lando. Only the + range endpoints are mapped to git; the commit count sizes the clone and is + capped so an old last-green can't produce an unbounded one. + +The agent gets the range (`base..head`), not a list of shas, and enumerates and +narrows it itself with `git log`. When the range isn't known to reach a green run +it is passed as `HEAD~N..HEAD` and the prompt stops asserting the culprit is in +it. + +`SOURCE_REF` / `SOURCE_DEPTH` pin the shallow clone to the failure commit, deep +enough to walk the range. The task's full and sanitized logs are written to files +for the agent to search. + +The fix stage writes a mozconfig mirroring the failing CI build (debug/opt, plus +asan/tsan/ccov), runs `mach bootstrap`, builds with the `build_firefox` tool, and +runs the failing tests with mach over Bash. The image ships Xvfb (started by the +entrypoint on `DISPLAY=:99`) with Firefox's runtime libraries and fonts, so GUI +harnesses run too, not just xpcshell and gtest. The container is Linux, so for a Windows or Mac failure the +run is read asymmetrically: a failure is real evidence the patch is wrong, while +a pass proves nothing about the failing platform and is reported as unverified. + +## Input + +- `FAILURE_TASKS` - a dictionary of failed Taskcluster test tasks + `{task_name: taskcluster_task_id}`. Everything else is resolved from the first + task id. + +## Output + +First stage - analysis (read-only): + +- `summary.md` - a short verdict +- `analysis.md` - detailed reasoning, with evidence from the logs and diffs +- `verdict.json` - `classification` (`regression` / `intermittent`), + `culprit_commit`, `culprit_bug`, `intermittent_bug`, `recommendation` + (`backout` / `land_fix` / `do_not_backout` / `rerun`) and `confidence` + +Second stage - fixing (only when a culprit was identified): + +- A patch in Hackbot format + +The result reports the `culprit_commit` so the caller can attribute the +regression to a developer. + +## Test the agent + +```sh +FAILURE_TASKS='{"test-linux1804-64/opt-xpcshell-1":"XyU4b_BIRdO_IeK6z_kcQg"}' \ + docker compose up test-repair-agent --build +``` + +Artifacts are written to `~/hackbot/artifacts/`. diff --git a/agents/test-repair/compose.yml b/agents/test-repair/compose.yml new file mode 100644 index 0000000000..cd22df2ad8 --- /dev/null +++ b/agents/test-repair/compose.yml @@ -0,0 +1,27 @@ +services: + test-repair-agent: + build: + context: ../.. + dockerfile: agents/test-repair/Dockerfile + target: agent + # Per-run inputs are interpolated by the listener/hackbot-api as env; pydantic + # AgentInputs validates them at runtime. BUGZILLA_MCP_URL is optional (Bugzilla + # is supplementary context) and left unset here. + environment: + - RUN_ID + - FAILURE_TASKS=${FAILURE_TASKS:-} + - SOURCE_REPO=/workspace/firefox + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:?error} + - WANDB_API_KEY=${WANDB_API_KEY:-} + # No uploader locally: summary/logs/artifacts are written under + # /artifacts/, bind-mounted to the host's ~/hackbot/artifacts. + - ARTIFACTS_DIR=/artifacts + # Firefox's content processes need far more than Docker's default 64MB + # /dev/shm; browser tests crash without this. + shm_size: 2gb + volumes: + - workspace:/workspace + - ${HOME}/hackbot/artifacts:/artifacts + +volumes: + workspace: diff --git a/agents/test-repair/docker-entrypoint.sh b/agents/test-repair/docker-entrypoint.sh new file mode 100644 index 0000000000..46fe1d49e2 --- /dev/null +++ b/agents/test-repair/docker-entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Firefox test harnesses (mochitest, reftest, wpt, marionette) need an X display; +# CI runs them under one. Without this only xpcshell and gtest could run locally. +set -e + +Xvfb "$DISPLAY" -screen 0 1920x1080x24 -nolisten tcp & + +# mach may start a test seconds after boot, so wait for the display to accept +# connections rather than racing it. +i=0 +while [ "$i" -lt 50 ]; do + xdpyinfo -display "$DISPLAY" >/dev/null 2>&1 && break + i=$((i + 1)) + sleep 0.2 +done + +exec "$@" diff --git a/agents/test-repair/hackbot.toml b/agents/test-repair/hackbot.toml new file mode 100644 index 0000000000..355221f344 --- /dev/null +++ b/agents/test-repair/hackbot.toml @@ -0,0 +1,10 @@ +[source] +repo_url = "https://github.com/mozilla-firefox/firefox.git" +checkout_path = "/workspace/firefox" +# The agent resolves the failure commit and candidate range from the task id at +# startup, then sets SOURCE_REF (the head/failure commit) and SOURCE_DEPTH (deep +# enough to reach the last-green commit) before the runtime prepares the tree. + +[firefox] +enabled = true +objdir = "objdir-test-repair" diff --git a/agents/test-repair/hackbot_agents/test_repair/__init__.py b/agents/test-repair/hackbot_agents/test_repair/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agents/test-repair/hackbot_agents/test_repair/__main__.py b/agents/test-repair/hackbot_agents/test_repair/__main__.py new file mode 100644 index 0000000000..85d763dc38 --- /dev/null +++ b/agents/test-repair/hackbot_agents/test_repair/__main__.py @@ -0,0 +1,85 @@ +import logging +import os +import tempfile +from pathlib import Path + +from hackbot_runtime import HackbotContext, run_async +from pydantic_settings import BaseSettings, SettingsConfigDict + +from .agent import TestRepairResult +from .logs import download_failure_logs +from .resolve import Investigation, resolve_investigation + +logger = logging.getLogger(__name__) + + +class AgentInputs(BaseSettings): + # Failing Taskcluster test tasks {task_name: task_id}. The agent resolves the + # push, last-green revision and candidate commit range from the task id. + failure_tasks: dict[str, str] + bugzilla_mcp_url: str = "" + model: str | None = None + max_turns: int | None = None + + # Compose passes unset per-run inputs as empty strings; treat those as absent. + model_config = SettingsConfigDict(extra="ignore", env_ignore_empty=True) + + +def _pin_checkout(investigation: Investigation) -> None: + """Pin the shallow clone to the failure commit, deep enough for the range. + + Read by the runtime when it prepares the source tree + (HackbotContext.source_repo). + """ + os.environ.setdefault("SOURCE_REF", investigation.failure_commit) + os.environ.setdefault("SOURCE_DEPTH", str(investigation.commit_range.span + 1)) + logger.info( + "Pinning checkout to %s with depth %s", + os.environ["SOURCE_REF"], + os.environ["SOURCE_DEPTH"], + ) + + +async def main(ctx: HackbotContext) -> TestRepairResult: + from .agent import run_test_repair + + inputs = AgentInputs() + if not inputs.failure_tasks: + raise ValueError("failure_tasks must contain at least one task") + + task_id = next(iter(inputs.failure_tasks.values())) + logger.info("Starting test-repair for task %s", task_id) + investigation: Investigation = resolve_investigation(task_id) + _pin_checkout(investigation) + + scratch_dir = Path(tempfile.mkdtemp(prefix="test-repair-")) + scratch_in = scratch_dir / "in" + scratch_out = scratch_dir / "out" + scratch_in.mkdir(parents=True, exist_ok=True) + scratch_out.mkdir(parents=True, exist_ok=True) + + logger.info("Downloading failure logs for %d task(s)", len(inputs.failure_tasks)) + task_logs = await download_failure_logs(inputs.failure_tasks, scratch_in) + + bugzilla_mcp_server = ( + {"type": "http", "url": inputs.bugzilla_mcp_url} + if inputs.bugzilla_mcp_url + else None + ) + return await run_test_repair( + bugzilla_mcp_server=bugzilla_mcp_server, + source_repo=ctx.source_repo, + fx_ctx=ctx.firefox, + investigation=investigation, + task_logs=task_logs, + scratch_out=scratch_out, + model=inputs.model, + max_turns=inputs.max_turns, + log=ctx.log_path, + verbose=True, + publish_file=ctx.publish_file, + ) + + +if __name__ == "__main__": + run_async(main) diff --git a/agents/test-repair/hackbot_agents/test_repair/agent.py b/agents/test-repair/hackbot_agents/test_repair/agent.py new file mode 100644 index 0000000000..04eb6acdd8 --- /dev/null +++ b/agents/test-repair/hackbot_agents/test_repair/agent.py @@ -0,0 +1,469 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Test-repair agent for Firefox CI test failures. + +Blame the commit that regressed a failing test and propose a fix. The pulse +listener only forwards failures that already passed its regression and flakiness +filters, so a regression is the prior, but the agent still reports the +classification it reaches from the logs. + +A two-stage claude-agent-sdk loop. Stage 1 (analysis, read-only) inspects the +candidate commit diffs and writes a verdict naming the culprit; Stage 2 (fix) +runs when a culprit is found and proposes a source patch, which the runtime +collects into ``changes.patch``. The :class:`TestRepairResult` is serialized into +``summary.json``'s ``findings`` and read by the notifier. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Literal + +from agent_tools import firefox +from agent_tools.claude_sdk import build_sdk_server +from agent_tools.firefox import FirefoxContext +from agent_tools.firefox.tools import bootstrap_firefox +from claude_agent_sdk import ( + ClaudeAgentOptions, + ClaudeSDKClient, + McpServerConfig, + ResultMessage, +) +from hackbot_runtime import AgentError, HackbotAgentResult +from hackbot_runtime.claude import Reporter + +from .config import ( + ADDITIONAL_DIRS, + ALLOWED_TOOLS, + ANALYSIS_MODEL, + BUGZILLA_READ_TOOLS, + FIREFOX_TOOLS, + FIX_MODEL, +) +from .logs import TaskLogs +from .prompts import ( + ANALYSIS_TEMPLATE, + CANDIDATE_INTRO_COMPLETE, + CANDIDATE_INTRO_PARTIAL, + ENVIRONMENT_NOTE, + FIX_TEMPLATE, + LAST_GREEN_LINE, + MAX_CANDIDATE_COMMITS, + MAX_TESTS_PER_GROUP, + VERIFY_LOCAL, + VERIFY_REMOTE, +) +from .resolve import CommitRange, Investigation + +_CLASSIFICATIONS = ("regression", "intermittent") +_RECOMMENDATIONS = ("backout", "do_not_backout", "land_fix", "rerun") +_SANITIZER_OPTIONS = {"asan": "address-sanitizer", "tsan": "thread-sanitizer"} + + +class TestRepairResult(HackbotAgentResult): + classification: Literal["regression", "intermittent"] + recommendation: Literal["backout", "do_not_backout", "land_fix", "rerun"] + culprit_commit: str | None = None + # Ranked commits that could not be ruled out, when no single culprit convinced + # the agent; lets sheriffs retrigger just these instead of backfilling. + candidate_commits: list[str] = [] + culprit_bug: int | None = None + confidence: float = 0.0 + last_green_revision: str | None = None + intermittent_bug: int | None = None + proposed_patch: bool = False + summary: str = "" + analysis: str = "" + + +def _build_options( + *, + model: str | None, + effort: str, + cwd: Path, + scratch_dir: Path, + mcp_servers: dict[str, McpServerConfig], + allowed_tools: list[str], + max_turns: int | None, +) -> ClaudeAgentOptions: + # The agent runs inside an isolated container, so tools run without + # per-command permission prompts. + return ClaudeAgentOptions( + model=model, + cwd=str(cwd), + mcp_servers=mcp_servers, + allowed_tools=allowed_tools, + disallowed_tools=["AskUserQuestion", "Task"], + add_dirs=[*ADDITIONAL_DIRS, str(scratch_dir)], + permission_mode="bypassPermissions", + effort=effort, + max_turns=max_turns, + setting_sources=[], + ) + + +async def _run_session( + reporter: Reporter, options: ClaudeAgentOptions, prompt: str +) -> ResultMessage | None: + result_msg: ResultMessage | None = None + async with ClaudeSDKClient(options=options) as client: + await client.query(prompt) + async for msg in client.receive_response(): + reporter.message(msg) + if isinstance(msg, ResultMessage): + result_msg = msg + return result_msg + + +def _check(result_msg: ResultMessage | None, stage: str) -> None: + if result_msg is None: + raise AgentError(f"{stage} stage produced no result message") + if result_msg.is_error: + raise AgentError( + f"{stage} stage failed: {result_msg.result or result_msg.subtype}" + ) + + +def _read_doc( + scratch_out: Path, + key: str, + publish_file: Callable[[str, Path, str | None], str] | None, +) -> str: + path = scratch_out / f"{key}.md" + if not path.exists(): + return "" + if publish_file is not None: + publish_file(f"{key}.md", path, "text/markdown") + return path.read_text() + + +def _read_verdict(scratch_out: Path) -> dict: + path = scratch_out / "verdict.json" + if not path.exists(): + return {} + try: + return json.loads(path.read_text()) + except (ValueError, OSError): + return {} + + +def _coerce_classification(value) -> str: + return value if value in _CLASSIFICATIONS else "regression" + + +def _coerce_recommendation(value, classification: str, has_culprit: bool) -> str: + if value not in _RECOMMENDATIONS: + value = "backout" if classification == "regression" else "do_not_backout" + if value == "backout" and not has_culprit: + return "do_not_backout" + return value + + +def _resolve_culprit(source_repo: Path, sha) -> str | None: + """Normalize a model-authored sha to a full commit hash in the checkout. + + The shallow clone holds exactly the candidate range, so a sha git cannot + resolve there was invented or is out of range; either way it is dropped. + """ + if not isinstance(sha, str) or not sha.strip(): + return None + sha = sha.strip() + try: + proc = subprocess.run( + [ + "git", + "-C", + str(source_repo), + "rev-parse", + "--verify", + f"{sha}^{{commit}}", + ], + capture_output=True, + text=True, + ) + full = proc.stdout.strip() if proc.returncode == 0 else "" + except OSError: + full = "" + if not full: + print(f"[test-repair] discarding culprit {sha!r}", file=sys.stderr) + return None + return full + + +def _resolve_candidates(source_repo: Path, value, culprit: str | None) -> list[str]: + """Validate the model's ranked fallback candidates, preserving their order. + + Same discard rule as the culprit: a sha git cannot resolve in the shallow clone + is not in the range. The culprit is dropped so the list stays a strict + alternative to it rather than repeating it. + """ + if not isinstance(value, list): + return [] + resolved: list[str] = [] + for sha in value[: MAX_CANDIDATE_COMMITS * 2]: + full = _resolve_culprit(source_repo, sha) + if full and full != culprit and full not in resolved: + resolved.append(full) + if len(resolved) == MAX_CANDIDATE_COMMITS: + break + return resolved + + +def _write_mozconfig(fx_ctx: FirefoxContext, investigation: Investigation) -> None: + """Write a mozconfig mirroring the failing CI build. + + ``build_firefox`` fails outright without one. The variant is mirrored because + a plain build cannot trigger an assertion, sanitizer or coverage failure. + + Always overwritten: ``/workspace`` is a persistent volume shared with the + other agents, so a leftover mozconfig from a previous run points the build at + a foreign objdir with foreign flags. + """ + options = ["ac_add_options --enable-application=browser"] + if investigation.debug_build: + options.append("ac_add_options --enable-debug") + else: + options += [ + "ac_add_options --disable-debug", + "ac_add_options --enable-optimize", + ] + if investigation.sanitizer: + options += [ + f"ac_add_options --enable-{_SANITIZER_OPTIONS[investigation.sanitizer]}", + "ac_add_options --disable-jemalloc", + ] + if investigation.coverage_build: + options.append("ac_add_options --enable-coverage") + options.append(f"mk_add_options MOZ_OBJDIR={fx_ctx.objdir}") + fx_ctx.mozconfig.write_text("\n".join(options) + "\n") + + +async def _bootstrap(fx_ctx: FirefoxContext) -> None: + """Install the build toolchain before the fix stage needs it. + + Deterministic prep rather than a tool call: it is slow and unconditional, so + spending agent turns deciding to run it wastes budget. Idempotent, and a + failure here is not fatal -- build_firefox reports its own errors. + """ + result = await bootstrap_firefox(fx_ctx.source_dir) + if not result.get("success"): + print(f"[test-repair] bootstrap: {result.get('message')}", file=sys.stderr) + + +def _as_float(value, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _as_int(value) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _assemble_result( + scratch_out: Path, + *, + verdict: dict, + source_repo: Path, + last_green_revision: str | None, + total_turns: int, + total_cost: float, + publish_file: Callable[[str, Path, str | None], str] | None, +) -> TestRepairResult: + classification = _coerce_classification(verdict.get("classification")) + culprit_commit = _resolve_culprit(source_repo, verdict.get("culprit_commit")) + recommendation = _coerce_recommendation( + verdict.get("recommendation"), classification, bool(culprit_commit) + ) + return TestRepairResult( + classification=classification, + recommendation=recommendation, + culprit_commit=culprit_commit, + candidate_commits=_resolve_candidates( + source_repo, verdict.get("candidate_commits"), culprit_commit + ), + culprit_bug=_as_int(verdict.get("culprit_bug")), + intermittent_bug=_as_int(verdict.get("intermittent_bug")), + confidence=_as_float(verdict.get("confidence")), + last_green_revision=last_green_revision, + proposed_patch=bool(verdict.get("proposed_patch")), + summary=_read_doc(scratch_out, "summary", publish_file), + analysis=_read_doc(scratch_out, "analysis", publish_file), + num_turns=total_turns, + total_cost_usd=total_cost, + ) + + +def _range_expr(commit_range: CommitRange) -> str: + """A git revision range for ``git log``, falling back to the clone depth.""" + if commit_range.base: + return f"{commit_range.base}..{commit_range.head}" + return f"HEAD~{commit_range.span}..HEAD" + + +def _failing_tests(investigation: Investigation) -> str: + groups = investigation.failing_groups + if not groups: + if not investigation.group_based: + return ( + "- (this suite does not report test manifests; identify the failing" + " tests from the failure logs)" + ) + return "- (failing groups could not be resolved; identify them from the logs)" + lines = [] + for group in groups: + shown = group.tests[:MAX_TESTS_PER_GROUP] + extra = len(group.tests) - len(shown) + tests = ", ".join(shown) + (f", +{extra} more" if extra > 0 else "") + count = f"{len(group.tests)} failed" if group.tests else "tests not resolved" + lines.append(f"- {group.group} ({count}): {tests or 'n/a'}") + return "\n".join(lines) + + +async def run_test_repair( + *, + bugzilla_mcp_server: McpServerConfig | None, + source_repo: Path, + fx_ctx: FirefoxContext, + investigation: Investigation, + task_logs: dict[str, TaskLogs], + scratch_out: Path, + model: str | None = None, + max_turns: int | None = None, + verbose: bool = False, + log: Path | None = None, + publish_file: Callable[[str, Path, str | None], str] | None = None, +) -> TestRepairResult: + """Blame the commit that regressed a failing test and propose a fix.""" + commit_range = investigation.commit_range + failure_commit = investigation.failure_commit + print( + f"[test-repair] analyzing {investigation.hg_revision} at {failure_commit}", + file=sys.stderr, + ) + + firefox_server = build_sdk_server("firefox", fx_ctx, firefox.TOOLS) + mcp_servers: dict[str, McpServerConfig] = {"firefox": firefox_server} + allowed_tools = [*ALLOWED_TOOLS, *FIREFOX_TOOLS] + # Bugzilla is optional context (searching for a related bug); wire it only + # when a broker URL is provided. + if bugzilla_mcp_server: + mcp_servers["bugzilla"] = bugzilla_mcp_server + allowed_tools += BUGZILLA_READ_TOOLS + + failure_logs = "\n".join( + f"- {name}: sanitized failures at {tl.sanitized} (start here); " + f"full log at {tl.full}" + for name, tl in task_logs.items() + ) + last_green_line = ( + LAST_GREEN_LINE.format(last_green_revision=investigation.last_green_revision) + if investigation.last_green_revision + else "" + ) + range_expr = _range_expr(commit_range) + intro = ( + CANDIDATE_INTRO_COMPLETE if commit_range.complete else CANDIDATE_INTRO_PARTIAL + ) + analysis_prompt = ANALYSIS_TEMPLATE.format( + failing_tests=_failing_tests(investigation), + harness=investigation.harness, + platform=investigation.platform or "unknown", + label=investigation.label or "unknown", + source_repo=source_repo, + failure_commit=failure_commit, + candidate_intro=intro.format(commit_range=range_expr, span=commit_range.span), + commit_range=range_expr, + max_candidates=MAX_CANDIDATE_COMMITS, + last_green_line=last_green_line, + failure_logs=failure_logs, + scratch_out=scratch_out, + ) + + total_cost = 0.0 + total_turns = 0 + scratch_dir = scratch_out.parent + + label = ( + investigation.failing_groups[0].group + if investigation.failing_groups + else investigation.hg_revision[:12] + ) + with Reporter(verbose=verbose, log_path=log) as reporter: + reporter.header(f"{label}: analysis") + analysis_opts = _build_options( + model=model or ANALYSIS_MODEL, + effort="high", + cwd=source_repo, + scratch_dir=scratch_dir, + mcp_servers=mcp_servers, + allowed_tools=allowed_tools, + max_turns=max_turns, + ) + result_msg = await _run_session(reporter, analysis_opts, analysis_prompt) + _check(result_msg, "analysis") + total_cost += result_msg.total_cost_usd or 0.0 + total_turns += result_msg.num_turns or 0 + + # Stage 2 (fix) runs only when the analysis blamed a real commit. + verdict = _read_verdict(scratch_out) + culprit_commit = _resolve_culprit(source_repo, verdict.get("culprit_commit")) + if culprit_commit: + reporter.header(f"{label}: fix") + _write_mozconfig(fx_ctx, investigation) + await _bootstrap(fx_ctx) + template = VERIFY_LOCAL if investigation.is_linux else VERIFY_REMOTE + verify_step = template.format( + harness=investigation.harness, platform=investigation.platform + ) + ENVIRONMENT_NOTE.format(platform=investigation.platform) + fix_prompt = FIX_TEMPLATE.format( + culprit_commit=culprit_commit, + verify_step=verify_step, + source_repo=source_repo, + scratch_out=scratch_out, + ) + fix_opts = _build_options( + model=model or FIX_MODEL, + effort="low", + cwd=source_repo, + scratch_dir=scratch_dir, + mcp_servers=mcp_servers, + allowed_tools=allowed_tools, + max_turns=max_turns, + ) + # A failed fix stage must not discard the analysis we already paid for. + try: + result_msg = await _run_session(reporter, fix_opts, fix_prompt) + if result_msg is not None: + total_cost += result_msg.total_cost_usd or 0.0 + total_turns += result_msg.num_turns or 0 + _check(result_msg, "fix") + except Exception as exc: + print(f"[test-repair] fix stage failed: {exc}", file=sys.stderr) + # Merge, since the fix stage may rewrite verdict.json without the culprit. + fix_verdict = _read_verdict(scratch_out) + verdict = { + **verdict, + **{k: v for k, v in fix_verdict.items() if v is not None}, + } + + return _assemble_result( + scratch_out, + verdict=verdict, + source_repo=source_repo, + last_green_revision=investigation.last_green_revision, + total_turns=total_turns, + total_cost=total_cost, + publish_file=publish_file, + ) diff --git a/agents/test-repair/hackbot_agents/test_repair/config.py b/agents/test-repair/hackbot_agents/test_repair/config.py new file mode 100644 index 0000000000..557dfb90e9 --- /dev/null +++ b/agents/test-repair/hackbot_agents/test_repair/config.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Models and tool allowlist for the test-repair agent.""" + +ANALYSIS_MODEL = "claude-opus-5" +FIX_MODEL = "claude-opus-5" + +# Bugzilla MCP tool names as exposed to the agent (mcp____). +BUGZILLA_READ_TOOLS = [ + "mcp__bugzilla__search_bugs", + "mcp__bugzilla__get_bugs", + "mcp__bugzilla__get_bug_comments", + "mcp__bugzilla__get_bug_attachments", + "mcp__bugzilla__download_attachment", +] + +# In-process Firefox build MCP tool (reused from agent-tools). Only the build +# tool: evaluate_testcase / evaluate_js_shell do not run the harnesses CI runs, +# so the agent runs the failing test itself with mach over Bash, and bootstrap is +# deterministic prep rather than something the agent has to decide to call. +BUILD_TOOL = "mcp__firefox__build_firefox" +FIREFOX_TOOLS = [BUILD_TOOL] + +# Built-in tools the agent may call alongside the MCP servers. The agent runs in +# an isolated container (permission_mode="bypassPermissions"), and reads the +# candidate commit diffs via Bash `git show`. +ALLOWED_TOOLS = [ + "Read", + "Grep", + "Glob", + "Bash", + "Edit", + "Write", + "MultiEdit", + "WebFetch", + "WebSearch", +] + +ADDITIONAL_DIRS = [ + "~/.mozbuild", + "~/.cache/uv/", +] diff --git a/agents/test-repair/hackbot_agents/test_repair/logs.py b/agents/test-repair/hackbot_agents/test_repair/logs.py new file mode 100644 index 0000000000..bebbc6cb03 --- /dev/null +++ b/agents/test-repair/hackbot_agents/test_repair/logs.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Download and sanitize a failing test task's log. + +Before invoking Claude we fetch each failing task's latest-run +``live_backing.log`` and write two files to the scratch dir: the full log and a +sanitized companion that keeps only the interesting lines -- the test harness's +``TEST-UNEXPECTED-*`` result lines plus ``ERROR -`` / ``FATAL -`` lines. The +agent starts from the sanitized log so its context isn't drowned by tens of MB +of output, and falls back to the full log for surrounding detail. +""" + +from __future__ import annotations + +import asyncio +import logging +import re +from pathlib import Path +from typing import NamedTuple + +import requests + +logger = logging.getLogger(__name__) + +ARTIFACT_URL = ( + "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/" + "task/{task_id}/artifacts/public/logs/live_backing.log" +) +_HEADERS = {"User-Agent": "hackbot-test-repair/1.0"} +_TIMEOUT = 120 +_MAX_LINES = 2000 + +_INTERESTING_RE = re.compile(r"TEST-UNEXPECTED-|(?:ERROR|FATAL) -") + + +class TaskLogs(NamedTuple): + """Paths to the two log files written for one failing task.""" + + sanitized: Path + full: Path + + +def _safe_filename(task_name: str) -> str: + return re.sub(r"[^A-Za-z0-9._-]+", "_", task_name).strip("_") or "task" + + +def sanitize_log(text: str) -> str: + """Keep only failure/error lines, deduping consecutive repeats and capping size.""" + kept: list[str] = [] + previous: str | None = None + for line in text.splitlines(): + if not _INTERESTING_RE.search(line): + continue + stripped = line.rstrip() + if stripped == previous: + continue + previous = stripped + kept.append(stripped) + if len(kept) >= _MAX_LINES: + kept.append(f"... (truncated at {_MAX_LINES} lines)") + break + return "\n".join(kept) + + +def _fetch_and_write(task_name: str, task_id: str, dest_dir: Path) -> TaskLogs: + safe = _safe_filename(task_name) + full_path = dest_dir / f"{safe}.log" + sanitized_path = dest_dir / f"{safe}.failures.txt" + url = ARTIFACT_URL.format(task_id=task_id) + try: + resp = requests.get(url, headers=_HEADERS, timeout=_TIMEOUT) + resp.raise_for_status() + full_path.write_text(resp.text) + sanitized = sanitize_log(resp.text) + sanitized_path.write_text( + sanitized if sanitized else f"(no failure lines matched in {url})\n" + ) + except requests.exceptions.RequestException as exc: + logger.warning("Failed to download log for %s (%s): %s", task_name, url, exc) + note = f"(failed to download {url}: {exc})\n" + full_path.write_text(note) + sanitized_path.write_text(note) + return TaskLogs(sanitized=sanitized_path, full=full_path) + + +async def download_failure_logs( + failure_tasks: dict[str, str], dest_dir: Path +) -> dict[str, TaskLogs]: + """Download and sanitize each task's log concurrently. + + Returns a mapping of task name to its :class:`TaskLogs`. + """ + names = list(failure_tasks) + logs = await asyncio.gather( + *( + asyncio.to_thread(_fetch_and_write, name, failure_tasks[name], dest_dir) + for name in names + ) + ) + return dict(zip(names, logs)) diff --git a/agents/test-repair/hackbot_agents/test_repair/prompts.py b/agents/test-repair/hackbot_agents/test_repair/prompts.py new file mode 100644 index 0000000000..c6224d43bb --- /dev/null +++ b/agents/test-repair/hackbot_agents/test_repair/prompts.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Prompt templates for the test-repair agent.""" + +# Failing tests listed per group before the list is elided. High enough to list +# them all in practice; only a whole-manifest failure hits it, and there the count +# printed alongside carries the signal the names would. +MAX_TESTS_PER_GROUP = 100 + +# Candidate shas kept from the model's ranked list when it names no single culprit. +MAX_CANDIDATE_COMMITS = 5 + +ANALYSIS_TEMPLATE = """\ +You are investigating a failing Firefox CI test to find the commit that broke it. +The CI listener already filtered out known intermittents, so treat this as a +genuine regression unless the logs clearly show otherwise. + +Failing test groups (manifests) and the tests that failed in each: +{failing_tests} +Test harness: {harness} +Test platform: {platform} +Test task: {label} +The task label pins the build type, test variant and chunk, not just the OS -- a +failure can be specific to this exact configuration rather than to the platform. + +The source tree is at {source_repo} (your working directory), checked out at the +failure commit {failure_commit}. The log and scratch paths below are outside +it -- `cd {source_repo}` to return. Search it with `git grep`, never `grep -r`: +the tree is enormous and a recursive walk hits the Bash timeout. +{candidate_intro} +{last_green_line} +Failure logs (start with the sanitized failures file; fall back to the full log): +{failure_logs} + +Do the following: +1. Read the sanitized failure lines to understand exactly how the test failed. +2. Enumerate every candidate with `git log --oneline {commit_range}`. That full + list is the set of possibilities; path filtering only decides what you read + first, it does not clear anyone. `git log --oneline {commit_range} -- ` + on the failing test's directory and on the source it exercises gets you to the + likeliest few, so `git show` those first -- but plenty of real culprits touch + neither: build config and mozconfig changes, shared headers, dependency and + toolchain bumps, manifest or harness changes, and changes that only shift + timing or memory elsewhere. If none of the path-matching commits explains the + failure, go through the rest of the list before concluding that nothing does. + You may search Bugzilla for a related bug. +3. Write these files to {scratch_out}: + - summary.md: a short (2-4 sentence) verdict. + - analysis.md: the detailed reasoning, with evidence from the logs and diffs. + - verdict.json: an object with keys "classification" ("regression" or + "intermittent"), "culprit_commit" (a full sha copied verbatim from the + candidate list above, or null if none is convincing), "candidate_commits" + (see below), "culprit_bug" (integer or null), "intermittent_bug" (integer or + null; the bug already tracking this intermittent, if you found one), + "recommendation" ("backout", "land_fix", "do_not_backout" or "rerun") and + "confidence" (0.0-1.0). + +Set "candidate_commits" whenever you are not confident in a single culprit: up to +{max_candidates} full shas from the range, most to least likely, so sheriffs can +retrigger just those instead of backfilling the whole range. Include the commits +you could not rule out, not only the one you like best; leave it as an empty list +when "culprit_commit" is certain, and still fill it in when "culprit_commit" is +null but some commits are more suspicious than others. + +Use "rerun" when you cannot tell whether the failure is real -- infrastructure +noise, a timeout under load, or a failure no candidate plausibly explains -- and +a retrigger would settle it. Prefer it over blaming a commit you are unsure of. + +Never guess a sha: any that is not a real commit in the range above is discarded, +in "culprit_commit" and in "candidate_commits" alike, and a wrong blame is worse +than no blame. Use null when nothing convinces you. + +Do not edit any source files in this step. +""" + +# Picked by resolve.CommitRange.complete: how firmly the prompt may assert that +# the culprit is inside the range. +CANDIDATE_INTRO_COMPLETE = """\ +The {span} commits in `{commit_range}` landed since this test was last green -- +the culprit is one of them.""" + +CANDIDATE_INTRO_PARTIAL = """\ +`{commit_range}` is the {span} most recent commits. This range is NOT known to +reach back to a run where the test was green, so the culprit may predate it -- if +nothing in it plausibly caused the failure, say so in your analysis, set +"culprit_commit" to null and leave "candidate_commits" empty rather than naming +the least implausible commit in the range.""" + +# Explicitly an hg revision: everything else in the prompt is a git hash, and an +# unlabelled hg node just makes the agent try `git` commands that exit 128. +LAST_GREEN_LINE = ( + "The test was last green at hg revision {last_green_revision} (not a git" + " object; the base of the range above is its git equivalent).\n" +) + +FIX_TEMPLATE = """\ +You determined that commit {culprit_commit} regressed the failing test(s). +Propose a minimal source patch that fixes the failure. + +The source tree is at {source_repo} (your working directory); the scratch paths +below are outside it -- `cd {source_repo}` to return. Search it with `git grep`, +never `grep -r`: the tree is enormous and a recursive walk hits the Bash timeout. + +1. Make the smallest change that addresses the root cause you identified in + {scratch_out}/analysis.md. +{verify_step} +3. Update {scratch_out}/verdict.json in place, preserving every key it already + has: set "proposed_patch" to true if you made a fix, and set "recommendation" + to "land_fix" only if you are confident in the fix; otherwise keep "backout". + +Keep the patch minimal and focused on the regression. +""" + +# The agent's container is Linux. Both steps run the test; they differ only in +# what a passing run is allowed to prove. +VERIFY_LOCAL = """\ +2. Verify the fix: build with the build_firefox tool (a mozconfig matching the + failing CI build is already in place), then run the failing test(s) yourself + over Bash with mach -- consult the tree's own AGENTS.md / CLAUDE.md for how it + runs {harness} tests. They should pass.""" + +VERIFY_REMOTE = """\ +2. Build with the build_firefox tool (a mozconfig matching the failing CI build is + already in place), then still run the failing test(s) over Bash with mach -- + consult the tree's own AGENTS.md / CLAUDE.md for how it runs {harness} tests. + Read the outcome asymmetrically: this container is Linux and the failure is on + {platform}, so a failure here is real evidence the patch is wrong or breaks + Linux, while a pass proves nothing about {platform} -- the test may even be + skipped or absent here. Report a pass as "not verified" rather than as + verification, and do not set "land_fix" on the strength of it alone.""" + +# The image ships Xvfb, Firefox's runtime libraries and fonts, and bootstrap has +# already run, so GUI harnesses are runnable -- but it is Debian, not CI's image. +ENVIRONMENT_NOTE = """ + Environment: a Debian container with a virtual display (DISPLAY is already set, + Xvfb is running) and the build toolchain bootstrapped, so mach can build and + run GUI harnesses. It is not CI's image though, and CI ran {platform}, so a + test that cannot run here at all is "could not verify", not a failure -- say + which it was.""" diff --git a/agents/test-repair/hackbot_agents/test_repair/resolve.py b/agents/test-repair/hackbot_agents/test_repair/resolve.py new file mode 100644 index 0000000000..7dceda8d18 --- /dev/null +++ b/agents/test-repair/hackbot_agents/test_repair/resolve.py @@ -0,0 +1,383 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Resolve a failing Taskcluster test task into everything the agent needs. + +From a task id alone, derive the push it belongs to (project + hg revision), the +tests that failed, the revision at which those tests were last green on the same +platform, and the git range that landed since then. Only the range endpoints are +mapped to git -- the agent enumerates the commits between them with ``git log`` +in the checkout. The agent recomputes all of this itself so its only input is a +task id; the pulse listener uses the same public Taskcluster / mozci / hg-pushlog +/ lando lookups only to decide which failures are worth investigating. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass + +import mozci.push # noqa: F401 (imported so mozci registers its data sources) +import requests +from mozci import data +from mozci.errors import ParentPushNotFound +from mozci.push import MAX_DEPTH, Push +from mozci.task import Status, is_no_groups_suite + +logger = logging.getLogger(__name__) + +_TC_TASK_URL = "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{task_id}" +_LANDO_HG2GIT = "https://lando.moz.tools/api/hg2git/firefox/{rev}" +_HG_BASE = "https://hg.mozilla.org" +# Taskcluster ``project`` tag -> hg pushlog repository path. +_REPO_PATHS = { + "autoland": "integration/autoland", + "mozilla-central": "mozilla-central", + "mozilla-beta": "releases/mozilla-beta", + "mozilla-release": "releases/mozilla-release", + "try": "try", +} +_HEADERS = {"User-Agent": "hackbot-test-repair/1.0"} +_TIMEOUT = 30 +# Ancestor pushes to walk looking for a green run. Each step is a live, uncached +# group_summaries lookup, so raising this trades startup latency for a better range. +LAST_GREEN_MAX_DEPTH = MAX_DEPTH +# Cap on commits (not pushes) in the range, bounding the clone depth. +MAX_RANGE_COMMITS = 100 + + +@dataclass(frozen=True) +class FailingGroup: + """A failing test manifest and every test that failed in it.""" + + group: str + tests: list[str] + + +@dataclass(frozen=True) +class CommitRange: + """The git commit range to search for the culprit.""" + + # The failure commit; what the checkout is pinned to. + head: str + # The last-green commit, exclusive. None when unknown or outside the cap. + base: str | None + # Commits in the range; bounds the shallow clone depth. + span: int + # Whether the culprit is provably inside the range. + complete: bool + + +@dataclass +class Investigation: + """The resolved context for one test-repair run, derived from a task id.""" + + project: str + hg_revision: str + harness: str + # Taskcluster ``test-platform`` tag, e.g. "linux1804-64-qr/debug". + platform: str + failing_groups: list[FailingGroup] + last_green_revision: str | None + commit_range: CommitRange + # Full task label, e.g. "test-linux1804-64-qr/debug-mochitest-browser-chrome-1". + # Unlike ``platform`` it also carries the test variant and chunk, which is what + # distinguishes two runs of the same suite on the same OS and build type. + label: str = "" + # False for suites that report no test manifests (gtest, jittest, talos, ...), + # where ``failing_groups`` is empty by nature rather than because the lookup + # failed, and blame has to be anchored on the task as a whole. + group_based: bool = True + + @property + def failure_commit(self) -> str: + return self.commit_range.head + + @property + def debug_build(self) -> bool: + return "debug" in self.platform + + @property + def is_linux(self) -> bool: + """Whether the agent's Linux container is the same OS family.""" + return self.platform.startswith("linux") + + @property + def sanitizer(self) -> str | None: + """The sanitizer CI built with, if any; a plain build cannot trigger it.""" + for name in ("asan", "tsan"): + if f"-{name}" in self.platform: + return name + return None + + @property + def coverage_build(self) -> bool: + return "-ccov" in self.platform + + +def _get_json(url: str) -> dict: + resp = requests.get(url, headers=_HEADERS, timeout=_TIMEOUT) + resp.raise_for_status() + return resp.json() + + +def _hg_to_git(rev: str) -> str | None: + try: + return _get_json(_LANDO_HG2GIT.format(rev=rev)).get("git_hash") + except requests.exceptions.RequestException: + logger.warning("lando hg2git lookup failed for %s", rev) + return None + + +def _harness(tags: dict) -> str: + """The tests.firefox.dev harness key for a test task.""" + suite = tags.get("test-suite") or "" + label = tags.get("label") or "" + if "xpcshell" in suite or "xpcshell" in label: + return "xpcshell" + if "mochitest" in suite: + return "mochitest" + # ``kind`` is "test" for every Firefox test task, so it is a last resort. + return suite or tags.get("kind") or "unknown" + + +def _failing_groups(task_id: str) -> list[FailingGroup]: + """Failing test groups for a task, via mozci; empty on any error.""" + try: + by_group = data.handler.get("test_task_failure_types", task_id=task_id) + except Exception: + logger.exception("Could not read failing groups for task %s", task_id) + return [] + groups: list[FailingGroup] = [] + for group, fails in by_group.items(): + if not group or not fails: + continue + groups.append(FailingGroup(group=group, tests=[test for test, _type in fails])) + return groups + + +def _same_platform(task, platform: str) -> bool: + return task.platform == platform or platform in (task.label or "") + + +def _test_status(push: Push, group: str, tests: list[str], platform: str) -> str | None: + """'passed'/'failed'/None for ``tests`` of ``group`` on ``platform``. + + Restricted to the failing tests and the failing platform: ``GroupSummary`` + aggregates every platform and every test in the manifest, so a push where the + group ran green on Windows -- or where only other tests in it passed -- must + not anchor a last-green for a Linux-only failure. None means non-decisive + (the group did not run here, or is intermittent/unfinished). + """ + summary = push.group_summaries.get(group) + if summary is None: + return None + wanted = set(tests) + statuses = set() + for task in summary.tasks: + if not _same_platform(task, platform): + continue + for result in task.results: + if result.group != group: + continue + if result.ok: + statuses.add("passed") + continue + failed = {test for test, _type in task.failure_types.get(group, [])} + statuses.add("failed" if not wanted or wanted & failed else "passed") + if "failed" in statuses: + return "failed" + return "passed" if statuses else None + + +def _label_status(push: Push, label: str) -> str | None: + """'passed'/'failed'/None for a whole task label on ``push``. + + The fallback for suites that report no test manifests, where there is no group + to key on and the task is the finest granularity available. The label already + pins the platform, build type and variant, and ``label_summaries`` excludes + taskgraph-chunked tasks -- whose label covers different tests on each push -- + so only genuinely comparable runs are considered. + """ + summary = push.label_summaries.get(label) + if summary is None: + return None + if summary.status == Status.PASS: + return "passed" + if summary.status == Status.FAIL: + return "failed" + # INTERMITTENT: it both passed and failed here, so it cannot anchor a green. + return None + + +def _walk_ancestors( + branch: str, rev: str, status_of, max_depth: int = LAST_GREEN_MAX_DEPTH +) -> str | None: + """Most recent ancestor revision that ``status_of`` reports as 'passed'. + + Best effort: None when no green ancestor is found within ``max_depth``, the + failure was already there upstream, or mozci errors. + """ + try: + ancestor = Push(rev, branch=branch) + for _ in range(max_depth): + try: + ancestor = ancestor.parent + except ParentPushNotFound: + break + status = status_of(ancestor) + if status == "passed": + return ancestor.rev + if status == "failed": + return None + except Exception: + logger.exception("Could not determine last-green at %s", rev) + return None + + +def _last_green( + branch: str, + rev: str, + failing: FailingGroup, + platform: str, + max_depth: int = LAST_GREEN_MAX_DEPTH, +) -> str | None: + """Most recent ancestor revision where the failing tests were green.""" + return _walk_ancestors( + branch, + rev, + lambda push: _test_status(push, failing.group, failing.tests, platform), + max_depth, + ) + + +def _last_green_label( + branch: str, rev: str, label: str, max_depth: int = LAST_GREEN_MAX_DEPTH +) -> str | None: + """Most recent ancestor revision where the whole failing task was green.""" + return _walk_ancestors( + branch, rev, lambda push: _label_status(push, label), max_depth + ) + + +def _count_commits(pushes: dict) -> int: + """Total changesets across the pushlog pushes.""" + return sum(len(p.get("changesets") or []) for p in pushes.values()) + + +def _resolve_range( + project: str, + head_rev: str, + last_green_rev: str | None, + max_commits: int, +) -> CommitRange | None: + """Resolve ``(last_green_rev, head_rev]`` into git endpoints and a commit count. + + None when the head cannot be mapped: pinning the checkout to an older commit + would blame the wrong change. ``base`` is dropped when unknown or outside the + capped clone depth, which also marks the range incomplete. + """ + head_git = _hg_to_git(head_rev) + if not head_git: + logger.error("Could not resolve a git hash for head revision %s", head_rev) + return None + + path = _REPO_PATHS.get(project, project) + pushlog = f"{_HG_BASE}/{path}/json-pushes" + try: + if last_green_rev: + url = ( + f"{pushlog}?fromchange={last_green_rev}" + f"&tochange={head_rev}&full=1&version=2" + ) + else: + url = f"{pushlog}?changeset={head_rev}&full=1&version=2" + pushes = _get_json(url).get("pushes") or {} + except requests.exceptions.RequestException: + logger.exception( + "Failed to fetch %s pushlog (%s..%s)", project, last_green_rev, head_rev + ) + pushes = {} + + span = max(_count_commits(pushes), 1) + if not last_green_rev or not pushes: + return CommitRange(head_git, None, span, False) + + if span > max_commits: + logger.warning( + "Range %s..%s has %d commits; capping the clone to the newest %d", + last_green_rev, + head_rev, + span, + max_commits, + ) + return CommitRange(head_git, None, max_commits, False) + + base_git = _hg_to_git(last_green_rev) + if not base_git: + logger.warning("Could not resolve a git hash for last-green %s", last_green_rev) + return CommitRange(head_git, None, span, False) + return CommitRange(head_git, base_git, span, True) + + +def resolve_investigation( + task_id: str, *, max_commits: int = MAX_RANGE_COMMITS +) -> Investigation: + """Resolve a failing test task into its investigation context. + + Raises ``ValueError`` when the task has no revision (nothing to investigate). + """ + task = _get_json(_TC_TASK_URL.format(task_id=task_id)) + tags = task.get("tags") or {} + project = tags.get("project") or "autoland" + hg_revision = (task.get("payload") or {}).get("env", {}).get("GECKO_HEAD_REV") + if not hg_revision: + raise ValueError(f"task {task_id} has no GECKO_HEAD_REV") + logger.info("Resolved task %s: project=%s rev=%s", task_id, project, hg_revision) + + label = tags.get("label") or "" + group_based = not is_no_groups_suite(label) + groups = _failing_groups(task_id) if group_based else [] + logger.info( + "Failing groups: %s", + ", ".join(g.group for g in groups) + or ("suite reports none" if not group_based else "none resolved"), + ) + + platform = tags.get("test-platform") or "" + # Group-less suites have nothing finer than the task to compare across pushes. + # A grouped suite whose lookup failed gets no last-green rather than a + # label-level one: its tasks are chunked, so the same label covers different + # tests on each push and ``label_summaries`` deliberately omits them. + if groups: + last_green = _last_green(project, hg_revision, groups[0], platform) + elif not group_based and label: + last_green = _last_green_label(project, hg_revision, label) + else: + last_green = None + logger.info("Last-green revision: %s", last_green or "not found") + + commit_range = _resolve_range(project, hg_revision, last_green, max_commits) + if commit_range is None: + raise ValueError(f"could not resolve a git commit for task {task_id}") + logger.info( + "Range %s..%s spans %d commit(s), complete: %s", + commit_range.base or "(unknown)", + commit_range.head, + commit_range.span, + commit_range.complete, + ) + + return Investigation( + project=project, + hg_revision=hg_revision, + harness=_harness(tags), + platform=platform, + failing_groups=groups, + last_green_revision=last_green, + commit_range=commit_range, + label=label, + group_based=group_based, + ) diff --git a/agents/test-repair/pyproject.toml b/agents/test-repair/pyproject.toml new file mode 100644 index 0000000000..34270dc3c5 --- /dev/null +++ b/agents/test-repair/pyproject.toml @@ -0,0 +1,30 @@ +[project] +name = "hackbot-agent-test-repair" +version = "0.1.0" +description = "Cloud Run Job image that runs the test-repair agent for hackbot-api" +requires-python = ">=3.12" +dependencies = [ + "hackbot-runtime[claude-sdk]", + "agent-tools[bugzilla,firefox]", + "claude-agent-sdk>=0.1.30", + "mcp>=1.0.0", + "requests", + "mozci~=2.4.8", + # mozci imports zstandard eagerly (default cache serializer) but only declares + # it under its optional `cache` extra, so pull it in explicitly. + "zstandard~=0.25.0", +] + +[tool.uv.sources] +hackbot-runtime = { workspace = true } +agent-tools = { workspace = true } + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["hackbot_agents"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/agents/test-repair/tests/__init__.py b/agents/test-repair/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/agents/test-repair/tests/test_agent_loop.py b/agents/test-repair/tests/test_agent_loop.py new file mode 100644 index 0000000000..db35df4502 --- /dev/null +++ b/agents/test-repair/tests/test_agent_loop.py @@ -0,0 +1,556 @@ +import asyncio +import json +import subprocess +from types import SimpleNamespace + +from hackbot_agents.test_repair import agent +from hackbot_agents.test_repair.prompts import MAX_TESTS_PER_GROUP +from hackbot_agents.test_repair.resolve import ( + CommitRange, + FailingGroup, + Investigation, +) + + +def _result_msg(is_error=False): + return SimpleNamespace( + is_error=is_error, + total_cost_usd=0.1, + num_turns=3, + result="max turns" if is_error else None, + subtype=None, + ) + + +def _git_repo(path): + """A two-commit repo standing in for the pinned shallow checkout. + + The culprit is validated with ``git rev-parse``, so the tests need real + commits. Returns [base, head]. + """ + + def git(*args): + return subprocess.run( + ["git", "-C", str(path), *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + subprocess.run(["git", "init", "-q", str(path)], check=True, capture_output=True) + git("config", "user.email", "t@example.com") + git("config", "user.name", "T") + shas = [] + for name in ("base", "head"): + (path / f"{name}.txt").write_text(name) + git("add", "-A") + git("commit", "-qm", name) + shas.append(git("rev-parse", "HEAD")) + return shas + + +def _investigation( + head, + base, + complete=True, + platform="linux1804-64/opt", + groups=None, + group_based=True, + label="test-linux1804-64/opt-mochitest-browser-chrome-swr-1", +): + return Investigation( + project="autoland", + hg_revision="hgrev", + harness="mochitest", + platform=platform, + failing_groups=groups + if groups is not None + else [FailingGroup("dom/base/test/mochitest.ini", ["dom/base/test/a.js"])], + last_green_revision="greenhg", + commit_range=CommitRange(head=head, base=base, span=2, complete=complete), + label=label, + group_based=group_based, + ) + + +def _fx_ctx(tmp_path): + return SimpleNamespace( + source_dir=tmp_path / "src", + mozconfig=tmp_path / ".mozconfig", + objdir=tmp_path / "obj", + ) + + +def _run( + tmp_path, + verdicts, + monkeypatch, + complete=True, + results=None, + platform="linux1804-64/opt", + groups=None, + group_based=True, +): + repo = tmp_path / "src" + repo.mkdir() + base, head = _git_repo(repo) + scratch_out = tmp_path / "out" + scratch_out.mkdir() + calls = [] + + async def fake_session(reporter, options, prompt): + calls.append(prompt) + verdict = verdicts.pop(0) + if verdict is not None: + if verdict.get("culprit_commit") == "HEAD": + verdict = {**verdict, "culprit_commit": head} + if verdict.get("candidate_commits"): + verdict = { + **verdict, + "candidate_commits": [ + {"HEAD": head, "BASE": base}.get(sha, sha) + for sha in verdict["candidate_commits"] + ], + } + (scratch_out / "verdict.json").write_text(json.dumps(verdict)) + (scratch_out / "summary.md").write_text("the verdict") + (scratch_out / "analysis.md").write_text("the reasoning") + return _result_msg(is_error=bool(results and results.pop(0))) + + bootstrapped = [] + + async def fake_bootstrap(firefox_dir): + bootstrapped.append(firefox_dir) + return {"success": True} + + monkeypatch.setattr(agent, "_run_session", fake_session) + monkeypatch.setattr(agent, "bootstrap_firefox", fake_bootstrap) + monkeypatch.setattr(agent, "build_sdk_server", lambda *a, **k: {"type": "sdk"}) + + result = asyncio.run( + agent.run_test_repair( + bugzilla_mcp_server=None, + source_repo=repo, + fx_ctx=_fx_ctx(tmp_path), + investigation=_investigation( + head, + base if complete else None, + complete, + platform, + groups, + group_based, + ), + task_logs={}, + scratch_out=scratch_out, + verbose=False, + log=None, + ) + ) + return result, calls, head + + +def test_culprit_runs_fix_stage(tmp_path, monkeypatch): + result, calls, head = _run( + tmp_path, + [ + {"recommendation": "backout", "culprit_commit": "HEAD", "confidence": 0.9}, + { + "recommendation": "land_fix", + "culprit_commit": "HEAD", + "confidence": 0.9, + "proposed_patch": True, + }, + ], + monkeypatch, + ) + assert len(calls) == 2 # analysis + fix + assert result.classification == "regression" + assert result.culprit_commit == head + assert result.recommendation == "land_fix" + assert result.proposed_patch is True + assert result.last_green_revision == "greenhg" + assert result.num_turns == 6 + # The fix stage can only verify anything if a mozconfig exists. + assert (tmp_path / ".mozconfig").exists() + + +def test_no_culprit_skips_fix_stage(tmp_path, monkeypatch): + result, calls, _head = _run( + tmp_path, + [{"recommendation": "backout", "culprit_commit": None, "confidence": 0.3}], + monkeypatch, + ) + assert len(calls) == 1 # no fix stage + assert result.culprit_commit is None + assert result.proposed_patch is False + assert result.recommendation == "do_not_backout" + + +def test_hallucinated_culprit_is_discarded(tmp_path, monkeypatch): + result, calls, _head = _run( + tmp_path, + [{"recommendation": "backout", "culprit_commit": "deadbeefdeadbeef"}], + monkeypatch, + ) + assert len(calls) == 1 + assert result.culprit_commit is None + assert result.recommendation == "do_not_backout" + + +def test_fix_stage_verdict_rewrite_keeps_analysis_culprit(tmp_path, monkeypatch): + # The fix stage may rewrite the whole file; the culprit must survive that. + result, _calls, head = _run( + tmp_path, + [ + { + "recommendation": "backout", + "culprit_commit": "HEAD", + "culprit_bug": 123, + "confidence": 0.9, + }, + {"recommendation": "land_fix", "proposed_patch": True}, + ], + monkeypatch, + ) + assert result.culprit_commit == head + assert result.culprit_bug == 123 + assert result.confidence == 0.9 + assert result.recommendation == "land_fix" + + +def test_failed_fix_stage_still_publishes_analysis(tmp_path, monkeypatch): + result, calls, head = _run( + tmp_path, + [ + {"recommendation": "backout", "culprit_commit": "HEAD", "confidence": 0.9}, + None, + ], + monkeypatch, + results=[False, True], + ) + assert len(calls) == 2 + assert result.culprit_commit == head + assert result.recommendation == "backout" + assert result.analysis == "the reasoning" + + +def test_complete_range_prompt_gives_a_base_anchored_range(tmp_path, monkeypatch): + _result, calls, head = _run(tmp_path, [{"culprit_commit": None}], monkeypatch) + assert "culprit is one of them" in calls[0] + # The agent enumerates the range itself rather than being handed every sha. + assert "git log --oneline" in calls[0] + assert f"..{head}" in calls[0] + + +def test_incomplete_range_prompt_does_not_assert_the_culprit(tmp_path, monkeypatch): + _result, calls, _head = _run( + tmp_path, [{"culprit_commit": None}], monkeypatch, complete=False + ) + assert "may predate" in calls[0] + assert "culprit is one of them" not in calls[0] + # Without a last-green base the range falls back to the clone depth. + assert "HEAD~2..HEAD" in calls[0] + + +def test_assemble_defaults_on_empty_verdict(tmp_path): + out = tmp_path / "out" + out.mkdir() + result = agent._assemble_result( + out, + verdict={}, + source_repo=tmp_path, + last_green_revision=None, + total_turns=1, + total_cost=0.0, + publish_file=None, + ) + # A regression is assumed, but there is no culprit to back out. + assert result.classification == "regression" + assert result.recommendation == "do_not_backout" + + +def test_assemble_tolerates_malformed_verdict_fields(tmp_path): + # verdict.json is model-authored; bad fields must not crash a finished run. + out = tmp_path / "out" + out.mkdir() + result = agent._assemble_result( + out, + verdict={ + "recommendation": "backout", + "culprit_commit": "abc", + "confidence": "high", + "culprit_bug": "n/a", + }, + source_repo=tmp_path, + last_green_revision=None, + total_turns=1, + total_cost=0.0, + publish_file=None, + ) + assert result.confidence == 0.0 + assert result.culprit_bug is None + assert result.classification == "regression" + # "abc" resolves to no commit in the checkout, so the blame is dropped. + assert result.culprit_commit is None + + +def test_assemble_reports_intermittent_classification(tmp_path): + out = tmp_path / "out" + out.mkdir() + result = agent._assemble_result( + out, + verdict={"classification": "intermittent", "intermittent_bug": 42}, + source_repo=tmp_path, + last_green_revision=None, + total_turns=1, + total_cost=0.0, + publish_file=None, + ) + assert result.classification == "intermittent" + assert result.intermittent_bug == 42 + assert result.recommendation == "do_not_backout" + + +def test_coerce_recommendation_defaults_by_classification(): + assert agent._coerce_recommendation("bogus", "regression", True) == "backout" + assert ( + agent._coerce_recommendation("bogus", "intermittent", False) == "do_not_backout" + ) + assert agent._coerce_recommendation("land_fix", "regression", True) == "land_fix" + # "backout" is meaningless without a commit to back out. + assert agent._coerce_recommendation("backout", "regression", False) == ( + "do_not_backout" + ) + + +def test_resolve_culprit_normalizes_against_the_checkout(tmp_path): + repo = tmp_path / "src" + repo.mkdir() + _base, head = _git_repo(repo) + assert agent._resolve_culprit(repo, head) == head + assert agent._resolve_culprit(repo, head[:8]) == head + assert agent._resolve_culprit(repo, "deadbeefdeadbeefdeadbeef") is None + assert agent._resolve_culprit(repo, None) is None + assert agent._resolve_culprit(repo, " ") is None + + +def test_range_expr_prefers_the_last_green_base(): + anchored = CommitRange(head="h" * 40, base="b" * 40, span=5, complete=True) + assert agent._range_expr(anchored) == f"{'b' * 40}..{'h' * 40}" + assert agent._range_expr(CommitRange("h" * 40, None, 5, False)) == "HEAD~5..HEAD" + + +def test_both_stages_name_the_checkout_path(tmp_path, monkeypatch): + # Only the scratch dir used to be named, so the agent cd'd there and then had + # to hunt the filesystem for the tree -- in both stages. + _result, calls, _head = _run( + tmp_path, + [{"culprit_commit": "HEAD"}, {"proposed_patch": True}], + monkeypatch, + ) + repo = str(tmp_path / "src") + assert len(calls) == 2 + for prompt in calls: + assert repo in prompt + + +def test_both_stages_steer_tree_searches_to_git_grep(tmp_path, monkeypatch): + # An unbounded `grep -r` over the Firefox tree burns the whole Bash timeout. + _result, calls, _head = _run( + tmp_path, + [{"culprit_commit": "HEAD"}, {"proposed_patch": True}], + monkeypatch, + ) + assert len(calls) == 2 + for prompt in calls: + assert "`git grep`" in prompt + + +def test_all_failing_tests_are_listed(tmp_path, monkeypatch): + tests = [f"dom/base/test/test_{i}.js" for i in range(3)] + _result, calls, _head = _run( + tmp_path, + [{"culprit_commit": None}], + monkeypatch, + groups=[FailingGroup("dom/base/test/mochitest.ini", tests)], + ) + for test in tests: + assert test in calls[0] + + +def test_long_failing_test_lists_are_elided(tmp_path, monkeypatch): + tests = [f"dom/base/test/test_{i}.js" for i in range(MAX_TESTS_PER_GROUP + 5)] + _result, calls, _head = _run( + tmp_path, + [{"culprit_commit": None}], + monkeypatch, + groups=[FailingGroup("dom/base/test/mochitest.ini", tests)], + ) + assert "+5 more" in calls[0] + + +def test_group_less_suites_say_so_instead_of_reporting_a_lookup_failure( + tmp_path, monkeypatch +): + # gtest and friends report no manifests at all, which is not the same thing as + # mozci failing to resolve them. + _result, calls, _head = _run( + tmp_path, + [{"culprit_commit": None}], + monkeypatch, + groups=[], + group_based=False, + ) + assert "does not report test manifests" in calls[0] + assert "could not be resolved" not in calls[0] + + +def test_prompt_names_the_task_not_just_the_platform(tmp_path, monkeypatch): + # The variant and chunk live in the label, so a config-specific failure is only + # visible to the agent if the label is. + _result, calls, _head = _run(tmp_path, [{"culprit_commit": None}], monkeypatch) + assert "test-linux1804-64/opt-mochitest-browser-chrome-swr-1" in calls[0] + + +def test_prompt_treats_path_filtering_as_ordering_not_exclusion(tmp_path, monkeypatch): + _result, calls, _head = _run(tmp_path, [{"culprit_commit": None}], monkeypatch) + assert "it does not clear anyone" in calls[0] + assert "go through the rest of the list" in calls[0] + + +def test_candidate_commits_are_kept_when_no_culprit_convinces(tmp_path, monkeypatch): + result, _calls, head = _run( + tmp_path, + [{"culprit_commit": None, "candidate_commits": ["HEAD", "BASE"]}], + monkeypatch, + ) + assert result.culprit_commit is None + # Ranked order is the model's; both are real commits in the range. + assert result.candidate_commits[0] == head + assert len(result.candidate_commits) == 2 + + +def test_candidate_commits_drop_hallucinations_and_the_culprit(tmp_path): + repo = tmp_path / "src" + repo.mkdir() + base, head = _git_repo(repo) + # The culprit is not repeated in the list, invented shas go, duplicates collapse. + assert agent._resolve_candidates( + repo, [head, base, base, "deadbeefdeadbeefdeadbeef", 42], head + ) == [base] + + +def test_candidate_commits_tolerate_a_non_list(tmp_path): + repo = tmp_path / "src" + repo.mkdir() + _git_repo(repo) + assert agent._resolve_candidates(repo, "not a list", None) == [] + assert agent._resolve_candidates(repo, None, None) == [] + + +def test_candidate_commits_are_capped(tmp_path): + repo = tmp_path / "src" + repo.mkdir() + base, head = _git_repo(repo) + shas = [head, base] * 10 + resolved = agent._resolve_candidates(repo, shas, None) + assert len(resolved) <= agent.MAX_CANDIDATE_COMMITS + assert resolved == [head, base] + + +def test_rerun_recommendation_survives_without_a_culprit(tmp_path, monkeypatch): + result, calls, _head = _run( + tmp_path, + [{"recommendation": "rerun", "culprit_commit": None, "confidence": 0.2}], + monkeypatch, + ) + assert len(calls) == 1 + assert result.recommendation == "rerun" + + +def test_linux_failure_expects_the_test_to_pass(tmp_path, monkeypatch): + _result, calls, _head = _run( + tmp_path, + [{"culprit_commit": "HEAD"}, {"proposed_patch": True}], + monkeypatch, + ) + assert "mach" in calls[1] + assert "They should pass." in calls[1] + assert "proves nothing" not in calls[1] + + +def test_non_linux_failure_still_runs_the_test_but_discounts_a_pass( + tmp_path, monkeypatch +): + # A failure on Linux is real evidence the patch is wrong; a pass says nothing + # about the failing platform, so it must not count as verification. + _result, calls, _head = _run( + tmp_path, + [{"culprit_commit": "HEAD"}, {"proposed_patch": True}], + monkeypatch, + platform="windows11-64-24h2/opt", + ) + assert "mach" in calls[1] + assert "proves nothing about windows11-64-24h2/opt" in calls[1] + assert "not verified" in calls[1] + + +def _mozconfig_for(tmp_path, platform): + tmp_path.mkdir(parents=True, exist_ok=True) + fx = _fx_ctx(tmp_path) + agent._write_mozconfig(fx, _investigation("h", None, platform=platform)) + return fx.mozconfig.read_text() + + +def test_mozconfig_mirrors_the_ci_build_variant(tmp_path): + opt = _mozconfig_for(tmp_path / "opt", "linux1804-64-qr/opt") + assert "--disable-debug" in opt and "--enable-optimize" in opt + assert "sanitizer" not in opt + + dbg = _mozconfig_for(tmp_path / "dbg", "linux1804-64-qr/debug") + assert "--enable-debug" in dbg + + # A plain build cannot trigger a sanitizer or coverage failure at all. + asan = _mozconfig_for(tmp_path / "asan", "linux1804-64-asan-qr/opt") + assert "--enable-address-sanitizer" in asan and "--disable-jemalloc" in asan + + tsan = _mozconfig_for(tmp_path / "tsan", "linux1804-64-tsan-qr/opt") + assert "--enable-thread-sanitizer" in tsan + + ccov = _mozconfig_for(tmp_path / "ccov", "linux1804-64-ccov/opt") + assert "--enable-coverage" in ccov + + +def test_verify_step_states_the_container_limits(tmp_path, monkeypatch): + _result, calls, _head = _run( + tmp_path, + [{"culprit_commit": "HEAD"}, {"proposed_patch": True}], + monkeypatch, + platform="linux1804-64-qr/debug", + ) + assert "virtual display" in calls[1] + assert "could not verify" in calls[1] + + +def test_last_green_is_labelled_as_an_hg_revision(tmp_path, monkeypatch): + # Every other revision in the prompt is a git hash; an unlabelled hg node + # makes the agent run git commands against it and get exit 128. + _result, calls, _head = _run(tmp_path, [{"culprit_commit": None}], monkeypatch) + assert "hg revision greenhg" in calls[0] + assert "not a git object" in calls[0] + + +def test_mozconfig_overwrites_a_foreign_one(tmp_path): + # /workspace is shared with the other agents, so a leftover mozconfig points + # the build at a foreign objdir with foreign flags. + tmp_path.mkdir(parents=True, exist_ok=True) + fx = _fx_ctx(tmp_path) + fx.mozconfig.write_text( + "ac_add_options --enable-release\n" + "mk_add_options MOZ_OBJDIR=/workspace/firefox/objdir-build-repair\n" + ) + agent._write_mozconfig(fx, _investigation("h", None, platform="linux1804-64/opt")) + written = fx.mozconfig.read_text() + assert "objdir-build-repair" not in written + assert "--enable-release" not in written + assert str(fx.objdir) in written diff --git a/agents/test-repair/tests/test_resolve.py b/agents/test-repair/tests/test_resolve.py new file mode 100644 index 0000000000..1e17a7dd66 --- /dev/null +++ b/agents/test-repair/tests/test_resolve.py @@ -0,0 +1,361 @@ +import pytest +from hackbot_agents.test_repair import resolve +from hackbot_agents.test_repair.resolve import FailingGroup +from mozci.errors import ParentPushNotFound +from mozci.task import Status + +GROUP = "dom/base/test/mochitest.ini" +TESTS = ["dom/base/test/test_a.js"] +PLATFORM = "linux1804-64-qr/debug" + + +def _investigation(**kwargs): + defaults = dict( + project="autoland", + hg_revision="hgrev", + harness="mochitest", + platform=PLATFORM, + failing_groups=[], + last_green_revision=None, + commit_range=resolve.CommitRange("head", None, 1, False), + ) + return resolve.Investigation(**{**defaults, **kwargs}) + + +class FakeResult: + def __init__(self, group, ok): + self.group = group + self.ok = ok + + +class FakeTask: + def __init__(self, platform, ok, failed_tests=()): + self.platform = platform + self.label = f"test-{platform}-mochitest-1" + self.results = [FakeResult(GROUP, ok)] + self.failure_types = {GROUP: [(t, "timeout") for t in failed_tests]} + + +class FakeSummary: + def __init__(self, *tasks): + self.tasks = list(tasks) + + +class FakeLabelSummary: + def __init__(self, status): + self.status = status + + +class FakePush: + def __init__(self, rev, summaries=None, parent=None, labels=None): + self.rev = rev + self.group_summaries = summaries or {} + self.label_summaries = labels or {} + self._parent = parent + + @property + def parent(self): + if self._parent is None: + raise ParentPushNotFound(f"no parent for {self.rev}") + return self._parent + + +def _failing(): + return FailingGroup(GROUP, TESTS) + + +def test_harness_detection(): + assert resolve._harness({"test-suite": "xpcshell"}) == "xpcshell" + assert resolve._harness({"label": "test-linux/opt-xpcshell-4"}) == "xpcshell" + assert resolve._harness({"test-suite": "mochitest-browser-chrome"}) == "mochitest" + # Every Firefox test task has kind=="test", so the suite must win over it. + assert ( + resolve._harness({"kind": "test", "test-suite": "web-platform-tests"}) + == "web-platform-tests" + ) + assert resolve._harness({}) == "unknown" + + +def test_platform_derived_flags(): + inv = _investigation(platform="linux1804-64-qr/debug") + assert inv.debug_build is True + assert inv.is_linux is True + win = _investigation(platform="windows11-64-24h2/opt") + assert win.debug_build is False + assert win.is_linux is False + + +def test_last_green_returns_first_passing_ancestor(monkeypatch): + green = FakePush("greenrev", {GROUP: FakeSummary(FakeTask(PLATFORM, ok=True))}) + head = FakePush("headrev", {}, parent=green) + monkeypatch.setattr(resolve, "Push", lambda rev, branch=None: head) + assert ( + resolve._last_green("autoland", "headrev", _failing(), PLATFORM) == "greenrev" + ) + + +def test_last_green_none_when_already_failing_upstream(monkeypatch): + parent = FakePush( + "parentrev", + {GROUP: FakeSummary(FakeTask(PLATFORM, ok=False, failed_tests=TESTS))}, + ) + head = FakePush("headrev", {}, parent=parent) + monkeypatch.setattr(resolve, "Push", lambda rev, branch=None: head) + assert resolve._last_green("autoland", "headrev", _failing(), PLATFORM) is None + + +def test_last_green_skips_pushes_that_only_ran_elsewhere(monkeypatch): + # The group passed on Windows but never ran on the failing Linux platform, so + # it cannot anchor a last-green -- the walk must continue past it. + green = FakePush("greenrev", {GROUP: FakeSummary(FakeTask(PLATFORM, ok=True))}) + other = FakePush( + "otherrev", + {GROUP: FakeSummary(FakeTask("windows11-64-24h2/opt", ok=True))}, + parent=green, + ) + head = FakePush("headrev", {}, parent=other) + monkeypatch.setattr(resolve, "Push", lambda rev, branch=None: head) + assert ( + resolve._last_green("autoland", "headrev", _failing(), PLATFORM) == "greenrev" + ) + + +def test_test_status_ignores_failures_of_other_tests(): + # The manifest failed, but not for the test we are investigating. + summary = FakeSummary( + FakeTask(PLATFORM, ok=False, failed_tests=["dom/base/test/test_other.js"]) + ) + push = FakePush("rev", {GROUP: summary}) + assert resolve._test_status(push, GROUP, TESTS, PLATFORM) == "passed" + assert resolve._test_status(push, GROUP, [], PLATFORM) == "failed" + + +def test_test_status_none_when_group_absent(): + assert resolve._test_status(FakePush("rev"), GROUP, TESTS, PLATFORM) is None + + +def test_label_status_maps_the_summary_status(): + label = "test-macosx1500-aarch64/opt-gtest-1proc" + passed = FakePush("rev", labels={label: FakeLabelSummary(Status.PASS)}) + failed = FakePush("rev", labels={label: FakeLabelSummary(Status.FAIL)}) + flaky = FakePush("rev", labels={label: FakeLabelSummary(Status.INTERMITTENT)}) + assert resolve._label_status(passed, label) == "passed" + assert resolve._label_status(failed, label) == "failed" + # Both passed and failed here, so it cannot anchor a green. + assert resolve._label_status(flaky, label) is None + assert resolve._label_status(FakePush("rev"), label) is None + + +def test_last_green_label_walks_to_a_green_task(monkeypatch): + # gtest reports no manifests, so the whole task is the finest granularity. + label = "test-macosx1500-aarch64/opt-gtest-1proc" + green = FakePush("greenrev", labels={label: FakeLabelSummary(Status.PASS)}) + # Did not run here at all: non-decisive, so the walk must continue. + absent = FakePush("absentrev", labels={}, parent=green) + head = FakePush("headrev", labels={}, parent=absent) + monkeypatch.setattr(resolve, "Push", lambda rev, branch=None: head) + assert resolve._last_green_label("autoland", "headrev", label) == "greenrev" + + +def test_last_green_label_none_when_already_failing_upstream(monkeypatch): + label = "test-macosx1500-aarch64/opt-gtest-1proc" + parent = FakePush("parentrev", labels={label: FakeLabelSummary(Status.FAIL)}) + head = FakePush("headrev", labels={}, parent=parent) + monkeypatch.setattr(resolve, "Push", lambda rev, branch=None: head) + assert resolve._last_green_label("autoland", "headrev", label) is None + + +def test_last_green_fails_soft_on_error(monkeypatch): + def boom(rev, branch=None): + raise RuntimeError("mozci exploded") + + monkeypatch.setattr(resolve, "Push", boom) + assert resolve._last_green("autoland", "headrev", _failing(), PLATFORM) is None + + +def _hg2git(rev): + return {"hgA": "gitA", "hgB": "gitB", "hgC": "gitC"}.get(rev) + + +def test_resolve_range_maps_only_the_endpoints(monkeypatch): + pushes = {"2": {"changesets": [{"node": "hgB"}, {"node": "hgC"}]}} + looked_up = [] + + def fake_hg_to_git(rev): + looked_up.append(rev) + return _hg2git(rev) + + monkeypatch.setattr(resolve, "_get_json", lambda url: {"pushes": pushes}) + monkeypatch.setattr(resolve, "_hg_to_git", fake_hg_to_git) + rng = resolve._resolve_range("autoland", "hgC", "hgA", 100) + assert (rng.head, rng.base, rng.span, rng.complete) == ("gitC", "gitA", 2, True) + # Two lando lookups regardless of range width; no per-commit mapping to fail. + assert looked_up == ["hgC", "hgA"] + + +def test_resolve_range_without_last_green_is_incomplete(monkeypatch): + pushes = {"1": {"changesets": [{"node": "hgA"}]}} + monkeypatch.setattr(resolve, "_get_json", lambda url: {"pushes": pushes}) + monkeypatch.setattr(resolve, "_hg_to_git", _hg2git) + rng = resolve._resolve_range("autoland", "hgA", None, 100) + assert rng.head == "gitA" + assert rng.base is None + assert rng.complete is False + + +def test_resolve_range_capped_drops_the_base(monkeypatch): + pushes = {str(i): {"changesets": [{"node": f"hg{i}"}]} for i in range(1, 6)} + monkeypatch.setattr(resolve, "_get_json", lambda url: {"pushes": pushes}) + monkeypatch.setattr(resolve, "_hg_to_git", lambda rev: "gitC") + rng = resolve._resolve_range("autoland", "hgC", "hgA", 2) + # The base falls outside the capped clone, so it can no longer anchor it. + assert rng.base is None + assert rng.span == 2 + assert rng.complete is False + + +def test_resolve_range_none_when_head_unresolvable(monkeypatch): + monkeypatch.setattr(resolve, "_hg_to_git", lambda rev: None) + assert resolve._resolve_range("autoland", "hgHEAD", "hgOLD", 100) is None + + +def test_resolve_range_incomplete_when_base_unresolvable(monkeypatch): + pushes = {"1": {"changesets": [{"node": "hgB"}]}} + monkeypatch.setattr(resolve, "_get_json", lambda url: {"pushes": pushes}) + monkeypatch.setattr( + resolve, "_hg_to_git", lambda rev: "gitB" if rev == "hgB" else None + ) + rng = resolve._resolve_range("autoland", "hgB", "hgA", 100) + assert rng.head == "gitB" + assert rng.base is None + assert rng.complete is False + + +def test_resolve_range_survives_pushlog_failure(monkeypatch): + def boom(url): + raise resolve.requests.exceptions.RequestException("hg down") + + monkeypatch.setattr(resolve, "_get_json", boom) + monkeypatch.setattr(resolve, "_hg_to_git", lambda rev: "gitHEAD") + rng = resolve._resolve_range("autoland", "hgHEAD", "hgOLD", 100) + assert rng.head == "gitHEAD" + assert rng.base is None + assert rng.span == 1 + assert rng.complete is False + + +def test_resolve_investigation_assembles_context(monkeypatch): + task = { + "tags": { + "project": "autoland", + "test-suite": "mochitest-browser-chrome", + "test-platform": "linux1804-64-qr/debug", + }, + "payload": {"env": {"GECKO_HEAD_REV": "hghead"}}, + } + monkeypatch.setattr(resolve, "_get_json", lambda url: task) + monkeypatch.setattr( + resolve, "_failing_groups", lambda tid: [FailingGroup(GROUP, TESTS)] + ) + monkeypatch.setattr(resolve, "_last_green", lambda *a: "greenrev") + monkeypatch.setattr( + resolve, + "_resolve_range", + lambda *a: resolve.CommitRange("gitHead", "gitBase", 2, True), + ) + + inv = resolve.resolve_investigation("TASK") + assert inv.project == "autoland" + assert inv.hg_revision == "hghead" + assert inv.harness == "mochitest" + assert inv.platform == "linux1804-64-qr/debug" + assert inv.debug_build is True + assert inv.last_green_revision == "greenrev" + assert inv.failure_commit == "gitHead" + assert inv.commit_range.base == "gitBase" + assert inv.commit_range.complete is True + assert inv.commit_range.span == 2 + + +def test_resolve_investigation_anchors_group_less_suites_on_the_task(monkeypatch): + label = "test-macosx1500-aarch64/opt-gtest-1proc" + task = { + "tags": { + "project": "autoland", + "test-suite": "gtest", + "test-platform": "macosx1500-aarch64/opt", + "label": label, + }, + "payload": {"env": {"GECKO_HEAD_REV": "hghead"}}, + } + monkeypatch.setattr(resolve, "_get_json", lambda url: task) + + def no_group_lookup(task_id): + raise AssertionError("group lookup must be skipped for a group-less suite") + + monkeypatch.setattr(resolve, "_failing_groups", no_group_lookup) + monkeypatch.setattr( + resolve, "_last_green_label", lambda branch, rev, lbl: f"green-{lbl}" + ) + monkeypatch.setattr( + resolve, + "_resolve_range", + lambda *a: resolve.CommitRange("gitHead", "gitBase", 2, True), + ) + + inv = resolve.resolve_investigation("TASK") + assert inv.group_based is False + assert inv.failing_groups == [] + assert inv.label == label + assert inv.last_green_revision == f"green-{label}" + + +def test_resolve_investigation_keeps_group_level_last_green_for_grouped_suites( + monkeypatch, +): + task = { + "tags": { + "project": "autoland", + "test-suite": "mochitest-browser-chrome", + "test-platform": PLATFORM, + "label": f"test-{PLATFORM}-mochitest-browser-chrome-1", + }, + "payload": {"env": {"GECKO_HEAD_REV": "hghead"}}, + } + monkeypatch.setattr(resolve, "_get_json", lambda url: task) + monkeypatch.setattr( + resolve, "_failing_groups", lambda tid: [FailingGroup(GROUP, TESTS)] + ) + monkeypatch.setattr(resolve, "_last_green", lambda *a: "greenrev") + + def no_label_lookup(*a): + raise AssertionError("label-level last-green is only for group-less suites") + + monkeypatch.setattr(resolve, "_last_green_label", no_label_lookup) + monkeypatch.setattr( + resolve, + "_resolve_range", + lambda *a: resolve.CommitRange("gitHead", "gitBase", 2, True), + ) + + inv = resolve.resolve_investigation("TASK") + assert inv.group_based is True + assert inv.last_green_revision == "greenrev" + + +def test_resolve_investigation_requires_a_git_commit(monkeypatch): + task = { + "tags": {"project": "autoland"}, + "payload": {"env": {"GECKO_HEAD_REV": "hghead"}}, + } + monkeypatch.setattr(resolve, "_get_json", lambda url: task) + monkeypatch.setattr(resolve, "_failing_groups", lambda tid: []) + monkeypatch.setattr(resolve, "_resolve_range", lambda *a: None) + with pytest.raises(ValueError): + resolve.resolve_investigation("TASK") + + +def test_resolve_investigation_requires_revision(monkeypatch): + monkeypatch.setattr(resolve, "_get_json", lambda url: {"tags": {}, "payload": {}}) + with pytest.raises(ValueError): + resolve.resolve_investigation("TASK") diff --git a/agents/test-repair/tests/test_scaffold.py b/agents/test-repair/tests/test_scaffold.py new file mode 100644 index 0000000000..02281a5ffc --- /dev/null +++ b/agents/test-repair/tests/test_scaffold.py @@ -0,0 +1,81 @@ +import os + +from hackbot_agents.test_repair import logs +from hackbot_agents.test_repair.__main__ import _pin_checkout +from hackbot_agents.test_repair.agent import TestRepairResult +from hackbot_agents.test_repair.resolve import CommitRange, Investigation + + +def test_sanitize_log_keeps_failure_and_error_lines(): + raw = "\n".join( + [ + "INFO - starting test", + "TEST-UNEXPECTED-FAIL | dom/test_a.js | assertion failed", + "some noise", + "12:00 ERROR - linker error", + "TEST-PASS | dom/test_b.js", + "FATAL - crash", + ] + ) + out = logs.sanitize_log(raw).splitlines() + assert any("TEST-UNEXPECTED-FAIL" in line for line in out) + assert any("ERROR - linker error" in line for line in out) + assert any("FATAL - crash" in line for line in out) + # Passing/info lines are dropped. + assert all("TEST-PASS" not in line for line in out) + assert all("starting test" not in line for line in out) + + +def test_sanitize_log_dedupes_consecutive_repeats(): + raw = "\n".join(["TEST-UNEXPECTED-FAIL | x | boom"] * 3) + assert len(logs.sanitize_log(raw).splitlines()) == 1 + + +def _investigation(**kwargs): + defaults = dict( + project="autoland", + hg_revision="hgrev", + harness="mochitest", + platform="linux1804-64-qr/opt", + failing_groups=[], + last_green_revision="greenhg", + commit_range=CommitRange(head="headsha", base="basesha", span=3, complete=True), + ) + return Investigation(**{**defaults, **kwargs}) + + +def test_pin_checkout_sets_ref_and_depth(monkeypatch): + monkeypatch.delenv("SOURCE_REF", raising=False) + monkeypatch.delenv("SOURCE_DEPTH", raising=False) + _pin_checkout(_investigation()) + assert os.environ["SOURCE_REF"] == "headsha" + # Depth spans the 3 commits in the range plus the base's parent. + assert os.environ["SOURCE_DEPTH"] == "4" + + +def test_result_model_serializes_findings(): + result = TestRepairResult( + num_turns=5, + total_cost_usd=0.42, + classification="regression", + recommendation="backout", + culprit_commit="deadbeef", + confidence=0.8, + summary="broke test", + analysis="the diff removed a null check", + ) + findings = result.model_dump() + assert findings["classification"] == "regression" + assert findings["recommendation"] == "backout" + assert findings["culprit_commit"] == "deadbeef" + assert findings["proposed_patch"] is False + + +def test_intermittent_result_defaults(): + result = TestRepairResult( + num_turns=2, + classification="intermittent", + recommendation="do_not_backout", + ) + assert result.culprit_commit is None + assert result.proposed_patch is False diff --git a/docker-compose.yml b/docker-compose.yml index 7c804a95d4..76e3b4c1ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ include: - path: agents/autowebcompat-repro/compose.yml - path: agents/bug-fix/compose.yml - path: agents/build-repair/compose.yml + - path: agents/test-repair/compose.yml - path: agents/frontend-triage/compose.yml - path: agents/test-plan-generator/compose.yml diff --git a/libs/agent-tools/agent_tools/firefox/tools/bootstrap_firefox.py b/libs/agent-tools/agent_tools/firefox/tools/bootstrap_firefox.py index 86cdadbc6f..6d6137d9ef 100644 --- a/libs/agent-tools/agent_tools/firefox/tools/bootstrap_firefox.py +++ b/libs/agent-tools/agent_tools/firefox/tools/bootstrap_firefox.py @@ -30,8 +30,8 @@ async def bootstrap_firefox(firefox_dir: Path) -> dict[str, Any]: process = await asyncio.create_subprocess_exec( "./mach", - "bootstrap", "--no-interactive", + "bootstrap", "--application-choice=browser", cwd=firefox_dir, stdout=asyncio.subprocess.PIPE, diff --git a/services/hackbot-api/app/agents.py b/services/hackbot-api/app/agents.py index eb5aa804bb..0ea62cecfc 100644 --- a/services/hackbot-api/app/agents.py +++ b/services/hackbot-api/app/agents.py @@ -10,6 +10,7 @@ BuildRepairInputs, FrontendTriageInputs, TestPlanGeneratorInputs, + TestRepairInputs, ) @@ -79,6 +80,16 @@ def model_to_env(inputs: BaseModel) -> dict[str, str]: job_name="hackbot-agent-frontend-triage", input_schema=FrontendTriageInputs, ), + "test-repair": AgentSpec( + name="test-repair", + description=( + "Analyze a Firefox CI test failure: classify it as a regression or an " + "intermittent, blame the culprit commit, and propose a fix patch for " + "regressions." + ), + job_name="hackbot-agent-test-repair", + input_schema=TestRepairInputs, + ), "test-plan-generator": AgentSpec( name="test-plan-generator", description=( diff --git a/services/hackbot-api/app/schemas.py b/services/hackbot-api/app/schemas.py index 5e3d85751a..bf1327463e 100644 --- a/services/hackbot-api/app/schemas.py +++ b/services/hackbot-api/app/schemas.py @@ -107,6 +107,15 @@ class BuildRepairInputs(BaseModel): max_turns: int | None = None +class TestRepairInputs(BaseModel): + # Failing Taskcluster test tasks {task_name: task_id}. The agent resolves the + # push, the last-green revision and the candidate commit range itself from the + # task id (the listener only filters which failures are worth investigating). + failure_tasks: dict[str, str] + model: str | None = None + max_turns: int | None = None + + class FrontendTriageInputs(BaseModel): bug_id: int model: str | None = None diff --git a/services/hackbot-api/tests/test_agents.py b/services/hackbot-api/tests/test_agents.py index 1e782b9963..aa14ec6c5b 100644 --- a/services/hackbot-api/tests/test_agents.py +++ b/services/hackbot-api/tests/test_agents.py @@ -7,6 +7,7 @@ from app.schemas import ( BugFixInputs, BuildRepairInputs, + TestRepairInputs, ) from app.schemas import ( TestPlanGeneratorInputs as PlanGeneratorInputs, @@ -91,3 +92,29 @@ def test_test_plan_generator_registry_uses_default_env_serializer(): assert spec.build_env is None assert spec.job_name == "hackbot-agent-test-plan-generator" assert spec.input_schema is PlanGeneratorInputs + + +def test_test_repair_registry_entry(): + spec = AGENT_REGISTRY["test-repair"] + assert spec.build_env is None + assert spec.input_schema is TestRepairInputs + assert spec.job_name == "hackbot-agent-test-repair" + assert spec.auto_apply_actions is False + + +def test_test_repair_env_serialization(): + # The agent resolves the test, commit range and clone depth from the task id, + # so the only per-run input is the failing task mapping. + env = model_to_env( + TestRepairInputs( + failure_tasks={"test-linux1804-64/opt-xpcshell-1": "abc123"}, + ) + ) + assert json.loads(env["FAILURE_TASKS"]) == { + "test-linux1804-64/opt-xpcshell-1": "abc123" + } + + +def test_test_repair_inputs_require_failure_tasks(): + with pytest.raises(ValidationError): + TestRepairInputs(model="claude-opus-4-8") diff --git a/services/hackbot-ui/components/TriggerForm.tsx b/services/hackbot-ui/components/TriggerForm.tsx index 8c2dc6e5f7..abadee5a58 100644 --- a/services/hackbot-ui/components/TriggerForm.tsx +++ b/services/hackbot-ui/components/TriggerForm.tsx @@ -47,7 +47,9 @@ export function TriggerForm() { const isReproAgent = agent === "autowebcompat-repro"; const isBuildRepairAgent = agent === "build-repair"; + const isTestRepairAgent = agent === "test-repair"; const isTestPlanAgent = agent === "test-plan-generator"; + const needsFailureTasks = isBuildRepairAgent || isTestRepairAgent; async function onSubmit(e: React.FormEvent) { e.preventDefault(); @@ -59,13 +61,15 @@ export function TriggerForm() { const hasBugId = Number.isInteger(parsedBugId) && parsedBugId > 0; const hasBugData = isReproAgent && bugData.trim().length > 0; - if (isBuildRepairAgent) { - if (hasBugId) inputs.bug_id = parsedBugId; - if (!gitCommit.trim()) { - setError("Enter a git commit hash."); - return; + if (needsFailureTasks) { + if (isBuildRepairAgent) { + if (hasBugId) inputs.bug_id = parsedBugId; + if (!gitCommit.trim()) { + setError("Enter a git commit hash."); + return; + } + inputs.git_commit = gitCommit.trim(); } - inputs.git_commit = gitCommit.trim(); if (!failureTasks.trim()) { setError("Enter failure tasks as a JSON object."); return; @@ -90,7 +94,7 @@ export function TriggerForm() { return; } inputs.failure_tasks = parsedTasks; - inputs.run_try_push = runTryPush; + if (isBuildRepairAgent) inputs.run_try_push = runTryPush; } else if (isTestPlanAgent) { if (!featureName.trim()) { setError("Enter a feature name."); @@ -127,7 +131,7 @@ export function TriggerForm() { const n = Number.parseInt(maxTurns, 10); if (Number.isInteger(n) && n > 0) inputs.max_turns = n; } - if (!isBuildRepairAgent && effort.trim()) inputs.effort = effort.trim(); + if (!needsFailureTasks && effort.trim()) inputs.effort = effort.trim(); setSubmitting(true); try { @@ -143,11 +147,14 @@ export function TriggerForm() { const run = body as RunRef; const label = isBuildRepairAgent ? `commit ${gitCommit.trim().slice(0, 12)}` - : isTestPlanAgent - ? featureName.trim() - : hasBugId - ? `bug ${parsedBugId}` - : "inline report"; + : isTestRepairAgent + ? Object.keys(inputs.failure_tasks as Record)[0] ?? + "test failure" + : isTestPlanAgent + ? featureName.trim() + : hasBugId + ? `bug ${parsedBugId}` + : "inline report"; saveRun({ run_id: run.run_id, agent: run.agent, @@ -184,7 +191,7 @@ export function TriggerForm() { - {!isBuildRepairAgent && !isTestPlanAgent && ( + {!needsFailureTasks && !isTestPlanAgent && (
+ + )} -
- -