Use Treeherder CLI for CI agents

agents/build-repair/Dockerfile

@@ -17,6 +17,18 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,target=/app,rw \
     uv sync --locked --no-dev --no-editable --package hackbot-agent-build-repair
 
+# treeherder-cli (MPL-2.0) lets the agent query Firefox CI: what failed on a push,
+# when a failure started, how often a job passes. Built from source because upstream
+# only ships prebuilt Linux binaries for x86_64, and `cargo install` also works for an
+# arm64 developer build. A bookworm builder is safe against either python:3.12 base
+# (same glibc, or older building for newer); rustls means no OpenSSL to link.
+FROM rust:1-slim-bookworm AS treeherder-cli
+ARG TREEHERDER_CLI_VERSION=0.2.17
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/build/target \
+    CARGO_TARGET_DIR=/build/target \
+    cargo install treeherder-cli --version ${TREEHERDER_CLI_VERSION} --locked --root /out
+
 FROM python:3.12 AS base
 
 COPY --from=builder /opt/venv /opt/venv
@@ -36,6 +48,12 @@ RUN useradd --create-home --shell /bin/bash agent \
     && mkdir -p /workspace \
     && chown agent:agent /workspace
 
+# Not ~/.cargo/bin, where `mach bootstrap` installs the Rust toolchain at runtime:
+# that directory is rustup-managed and inside the agent's HOME, so a root-owned copy
+# made at build time does not belong in it. /usr/local/bin is on PATH for every user
+# and nothing at runtime writes to it.
+COPY --from=treeherder-cli /out/bin/treeherder-cli /usr/local/bin/treeherder-cli
+
 # `mach bootstrap` installs the toolchain here at runtime; put it on PATH so the
 # agent's own `./mach build` (and the build_firefox tool) find rustc/clang.
 ENV PATH="/home/agent/.cargo/bin:/home/agent/.mozbuild/clang/bin:${PATH}"

agents/build-repair/hackbot_agents/build_repair/__main__.py

@@ -2,7 +2,7 @@
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from .agent import BuildRepairResult, run_build_repair
-from .resolve import resolve_git_commits
+from .resolve import resolve_push
 
 
 class AgentInputs(BaseSettings):
@@ -32,7 +32,8 @@ async def main(ctx: HackbotContext) -> BuildRepairResult:
     # The first is the failure commit the tree is checked out at; the rest let
     # the agent blame the culprit.
     task_id = next(iter(inputs.failure_tasks.values()))
-    git_commits = resolve_git_commits(task_id, inputs.git_commit)
+    push = resolve_push(task_id, inputs.git_commit)
+    git_commits = push.git_commits
 
     # Pin the checkout to the failure commit and fetch deep enough to include the
     # whole push, so the agent can `git show` every commit in it.
@@ -47,6 +48,8 @@ async def main(ctx: HackbotContext) -> BuildRepairResult:
         fx_ctx=ctx.firefox,
         bug_id=inputs.bug_id,
         git_commits=git_commits,
+        project=push.project,
+        hg_revision=push.hg_revision,
         failure_tasks=inputs.failure_tasks,
         run_try_push=inputs.run_try_push,
         model=inputs.model,

agents/build-repair/hackbot_agents/build_repair/agent.py

@@ -33,7 +33,6 @@
     ToolUseBlock,
     UserMessage,
 )
-from hackbot_agents.build_repair.logs import download_failure_logs
 from hackbot_agents.build_repair.try_push import TRY_TOOLS
 from hackbot_runtime import AgentError, HackbotAgentResult
 from hackbot_runtime.claude import Reporter
@@ -56,8 +55,11 @@
     FIX_TEMPLATE,
     PUSH_COMMIT_LINE,
     PUSH_CONTEXT,
+    TREEHERDER_STEP,
+    TREEHERDER_STEP_NO_PUSH,
     TRY_PUSH_INSTRUCTIONS,
 )
+from .resolve import task_push
 
 TARGET_SOFTWARE = "Mozilla Firefox"
 
@@ -143,6 +145,8 @@ async def run_build_repair(
     fx_ctx: FirefoxContext,
     bug_id: int | None = None,
     git_commits: list[str],
+    project: str | None = None,
+    hg_revision: str | None = None,
     failure_tasks: dict[str, str],
     run_try_push: bool = False,
     model: str | None = None,
@@ -163,18 +167,9 @@ async def run_build_repair(
     print(f"[build_repair] repairing {label} at {failure_commit}", file=sys.stderr)
 
     scratch_dir = Path(tempfile.mkdtemp(prefix=f"build-repair-{bug_id or 'nobug'}-"))
-    scratch_in = scratch_dir / "in"
     scratch_out = scratch_dir / "out"
-    scratch_in.mkdir(parents=True, exist_ok=True)
     scratch_out.mkdir(parents=True, exist_ok=True)
 
-    task_logs = await download_failure_logs(failure_tasks, scratch_in)
-    failure_logs = "\n".join(
-        f"- {name}: sanitized errors at {tl.sanitized} (start here); "
-        f"full log at {tl.full}"
-        for name, tl in task_logs.items()
-    )
-
     firefox_tools = [*firefox.TOOLS, *TRY_TOOLS] if run_try_push else firefox.TOOLS
     firefox_server = build_sdk_server("firefox", fx_ctx, firefox_tools)
     mcp_servers: dict[str, McpServerConfig] = {
@@ -189,19 +184,38 @@ async def run_build_repair(
     ]
 
     task_name = next(iter(failure_tasks), "")
+    if not (project and hg_revision) and failure_tasks:
+        # The eval harness drives the agent from git commits alone, but the logs
+        # only come from Treeherder, so the push has to be resolved either way.
+        try:
+            project, hg_revision = task_push(next(iter(failure_tasks.values())))
+        except Exception:
+            print("[build_repair] could not resolve the push", file=sys.stderr)
+    treeherder_step = (
+        TREEHERDER_STEP.format(
+            project=project,
+            hg_revision=hg_revision,
+            task_name=task_name,
+            scratch_out=scratch_out,
+        )
+        if project and hg_revision
+        else TREEHERDER_STEP_NO_PUSH.format(scratch_out=scratch_out)
+    )
     analysis_prompt = ANALYSIS_TEMPLATE.format(
         target_software=TARGET_SOFTWARE,
         git_commit=failure_commit,
+        source_repo=source_repo,
         push_context=_push_context(git_commits),
+        treeherder_step=treeherder_step,
         blame_step=_blame_step(git_commits, scratch_out),
-        failure_logs=failure_logs,
         scratch_out=scratch_out,
         bug_context=BUG_CONTEXT.format(bug_id=bug_id) if bug_id is not None else "",
         bug_step=BUG_ANALYSIS_STEP.format(bug_id=bug_id) if bug_id is not None else "",
         logs_num=3 if bug_id is not None else 2,
     )
     fix_prompt = FIX_TEMPLATE.format(
         target_software=TARGET_SOFTWARE,
+        source_repo=source_repo,
         scratch_out=scratch_out,
         try_push=(
             TRY_PUSH_INSTRUCTIONS.format(task_name=task_name) if run_try_push else ""
@@ -231,6 +245,7 @@ async def run_build_repair(
             reporter, analysis_opts, analysis_prompt, captured, tracked
         )
         _check(result_msg, label, "analysis")
+        _check_blocked(scratch_out)
         total_cost += result_msg.total_cost_usd or 0.0
         total_turns += result_msg.num_turns or 0
 
@@ -316,6 +331,26 @@ async def _run_session(
     return result_msg
 
 
+def _check_blocked(scratch_out: Path) -> None:
+    """Fail the run unless the agent actually retrieved a failure log.
+
+    Analysing a failure whose log never arrived produces a confident verdict
+    invented from the diff alone (bug 6665), so a missing log is an error rather
+    than a verdict, and the error reaches the API and the UI. The agent is asked to
+    report the blocker itself in error.txt; the log check is what makes it a
+    guarantee rather than an instruction.
+    """
+    blocker = scratch_out / "error.txt"
+    if blocker.exists():
+        raise AgentError(blocker.read_text().strip() or "agent reported it was blocked")
+    logs = scratch_out / "logs"
+    if not any(logs.glob("**/*live_backing_log*")):
+        raise AgentError(
+            "no failure log was retrieved: treeherder-cli left nothing under "
+            f"{logs}, so any verdict would be guesswork"
+        )
+
+
 def _check(result_msg: ResultMessage | None, label: str, stage: str) -> None:
     if result_msg is None:
         raise AgentError(f"{label}: {stage} stage produced no result message")

agents/build-repair/hackbot_agents/build_repair/prompts.py

@@ -7,14 +7,15 @@
 
 ANALYSIS_TEMPLATE = """You are an expert {target_software} engineer tasked with analyzing and fixing a build failure.
 
-Investigate why the {target_software} build broke at commit {git_commit}. The source tree
-is already checked out at that commit (your working directory).
+Investigate why the {target_software} build broke at commit {git_commit}. The source
+tree is at {source_repo} (your working directory), checked out at that commit. Stay
+in it: write scratch files by absolute path rather than `cd`-ing elsewhere. If a
+command reports "not a git repository" you have moved -- run `git -C {source_repo}`
+rather than hunting for the tree.
 {push_context}{bug_context}
 Analyze the following:
 1. The git diff of commit {git_commit} (use `git show {git_commit}`).
-{bug_step}{logs_num}. The Taskcluster build failure logs. Each failing task has a sanitized log (only the ERROR -/FATAL - lines) and the full log. Start from the sanitized log -- it usually pinpoints the failing file and line. The full log can be tens of thousands of lines, so grep it for that file/line rather than reading it sequentially:
-{failure_logs}
-
+{bug_step}{logs_num}.{treeherder_step}
 Create these documents:
 1. {scratch_out}/analysis.md -- the developer's reference, readable in under a
    minute. Under 40 lines, in exactly these sections, each a short paragraph or a
@@ -44,6 +45,65 @@
 
 PUSH_COMMIT_LINE = "- {commit}"
 
+TREEHERDER_STEP = r"""\
+   The build failure logs, via `treeherder-cli`. This push is {project} revision
+   {hg_revision}, and the failing task is '{task_name}'. Start with the error lines:
+   `treeherder-cli {hg_revision} --repo {project} --filter '{task_name}'
+   --include-intermittent --fetch-logs --pattern '\b(?:ERROR|FATAL) -'
+   --cache-dir {scratch_out}/logs | head -100`
+   Anchor short patterns on a word boundary: bare `ERROR -` also matches inside
+   `-DHAVE_STRERROR -D...` and buries the real errors in compiler command lines.
+   Each hit prints as `live_backing_log:<line>` and the full logs stay under
+   {scratch_out}/logs, where those line numbers apply, so read a window around one
+   to get the whole diagnostic:
+   `sed -n '165280,165320p' {scratch_out}/logs/job_<id>/live_backing_log.log | cut -c1-200`
+   An `ERROR -` line is usually only the first line of a compiler error -- the
+   offending source and the `^` caret follow it. Never read or cat a whole log;
+   they run to six figures of lines.
+   Clip the width as well as the line count: log lines run to thousands of
+   characters, so append `| cut -c1-200` to any grep or sed over a log -- 40 wpt
+   lines alone came to 38 KB without it.
+   Two things can stop that command finding the job, and both are recoverable:
+   - Treeherder sometimes returns a malformed response ("error decoding response
+     body"). Retry the same command once; it usually succeeds.
+   - "N passing jobs ... no failures found" almost always means
+     `--include-intermittent` was missing: a failure a sheriff has already
+     classified is hidden without it, and that covers most of them. Check the flag
+     is there and re-run.
+   Once the retry has also failed there is nothing to analyse. Write what you ran
+   and what it reported to {scratch_out}/error.txt and stop: do not write the other
+   documents, guess a cause, or propose a fix from the diff alone. Do not fetch the
+   artifact from Taskcluster yourself either: after a retry or rerun the latest
+   artifact can be a passing run's log, so a wrong log is worse than none. The run
+   is meant to fail here.
+
+   The same command answers CI questions about the push. `--compare <revision>`
+   says whether the failure is new here or was already failing earlier;
+   `--lookback 50 --suspects` finds the push window a failure started in;
+   `--similar-history <job id>` gives a job's recent pass rate, which separates a
+   real bustage from infrastructure flakiness.
+
+   Every revision you pass has to be a real hg revision that treeherder-cli
+   printed. It rejects anything else with "No push found for revision" -- both the
+   git shas of the push commits above and placeholders like `parent`. To compare
+   against a neighbouring push, get its revision from `--context 3` first, which
+   lists the pushes either side of this one, then pass that to `--compare`.
+
+   `--help` lists the rest. The default markdown output is the compact one -- only
+   add `--json` when you will parse it. To see more of a log widen `--pattern`.
+   Always pass `--filter` and pipe through `head`: unfiltered, one push can print
+   hundreds of megabytes straight into your context. Never pass `--watch` or
+   `--stream-failures` -- they block until CI finishes.
+"""
+
+
+TREEHERDER_STEP_NO_PUSH = """\
+   This push could not be resolved on Treeherder, so the failure logs cannot be
+   reached. Write that to {scratch_out}/error.txt and stop: do not write the other
+   documents or propose a fix from the diff alone.
+"""
+
+
 BLAME_STEP = """4. {scratch_out}/blame.json naming the commit that introduced the failure, as JSON:
    {{"blamed_commit": "<full git sha>", "reason": "<one sentence>"}}. Use one of the
    push commits listed above when there are several, otherwise the checked-out
@@ -66,7 +126,17 @@
 1. {scratch_out}/analysis.md -- your analysis of what caused the failure
 2. {scratch_out}/planning.md -- your fixing plan
 
-Edit the source files in the working directory to repair the build. A mozconfig
+Edit the source files in {source_repo} (your working directory) to repair the build.
+Editing: use Edit on a file that already exists -- Write refuses until the file has
+been read, which costs a turn. To see how a commit handled comparable files, run
+`git show <sha> -- <dir>` rather than guessing a sibling's name.
+
+Working in this tree: review your own edits with `git diff --stat` and `git diff --
+<path>`, never `git status` -- after a build the objdir adds millions of untracked
+files and the output runs to tens of MB. Logs already fetched sit under
+{scratch_out}/logs; when you grep or sed one, cap the width as well as the line
+count (`| head -40 | cut -c1-200`), because a single build-log line can be 10 KB.
+ A mozconfig
 that mirrors the failing CI configuration (release milestone, warnings-as-errors)
 is already set up. Verify the fix compiles with the build_firefox tool, passing
 the directory of the file you changed as `target` (e.g. 'docshell/base') for a