feat: persist agent report on read result records (task 0032)

Retain the agent's final report on the benchmark result record for read
tasks, so a failed read is diagnosable directly from the stored record
instead of only carrying an opaque `incorrect` tag. The runner resolves
the scoring spec once and records `run.finalReport` when the spec is a
read; mutation records omit the field entirely. The sample store needs no
change — it serializes whatever record it is handed.

This is the prerequisite for confirming the read-open-issue-count failure
from real report text before the state-aware count-line change (task 0033).

bench/result.ts

@@ -72,6 +72,15 @@ export interface ResultRecord {
 
   /** Pass/fail outcome with a failure tag. */
   outcome: Outcome;
+
+  /**
+   * The agent's final report, retained for read tasks so a failed read is
+   * diagnosable directly from the record — the exact text the agent submitted,
+   * distinguishing a wrong answer from a right one phrased in words the checker's
+   * accepted-phrasing list did not match. Absent for mutation tasks, which are
+   * scored by diffing repository state and have no agent report to record.
+   */
+  report?: string;
 }
 
 /**

bench/runner.test.ts

@@ -6,6 +6,7 @@ import type { RepoState } from "./scoring-spec.js";
 import type { BenchAccess, RepoCoords } from "./seed.js";
 import { groundTruth } from "./seed-plan.js";
 import { createSampleStore } from "./store.js";
+import type { BenchTask } from "./task.js";
 import { SAMPLE_TASK } from "./task.js";
 import type { AgentDriver, BenchHost } from "./runner.js";
 import { runCell } from "./runner.js";
@@ -397,4 +398,110 @@ describe("runCell", () => {
     if (sample === undefined) return;
     expect(sample.outcome).toEqual({ pass: false, failure: "incorrect" });
   });
+
+  // Behavior: a completed read cell produces a record whose report is the agent's
+  // final report (benchmark-harness spec, record-assembly seam). A read task is
+  // scored by matching the agent's final report against required facts; the
+  // recorded ResultRecord must carry that final report in its `report` field so a
+  // read is diagnosable from the record alone. Here an inline read task's fact is
+  // satisfied by the driver's finalReport (it contains "5 open"), so the cell
+  // completes as a pass; the recorded sample's `report` must equal the exact
+  // planted finalReport literal — an independent literal, read back through the
+  // store, not recomputed from runner.ts.
+  it("records the agent's final report on a completed read cell", async () => {
+    const READ_TASK: BenchTask = {
+      id: "count-open-issues",
+      tier: "read",
+      intent: "How many issues are open?",
+      scoringSpec: () => ({
+        kind: "read",
+        facts: [{ description: "open-issue count", anyOf: ["5 open"] }],
+      }),
+    };
+
+    // The independent literal we plant as the driver's final report. It contains
+    // the fact's rendering ("5 open"), so the read scorer scores it a pass.
+    const FINAL_REPORT = "There are 5 open issues.";
+
+    const readDriver: AgentDriver = {
+      async run() {
+        return {
+          tokens: { freshInput: 50, cacheCreation: 0, cacheRead: 0, output: 10 },
+          turns: 2,
+          imputedCostUsd: 0.03,
+          transcript: [{ kind: "mcp", server: "gitea-mcp", tool: "list_repo_issues" }],
+          finalReport: FINAL_REPORT,
+          stoppedByTurnCap: false,
+        };
+      },
+    };
+
+    const { host } = createFakeHost();
+    const store = createSampleStore(storeRoot);
+
+    const outcome = await runCell({
+      arm: "gitea-mcp",
+      task: READ_TASK,
+      trial: 1,
+      access: ACCESS,
+      host,
+      driver: readDriver,
+      store,
+      bounds: { turnCap: 10, wallClockMs: 60_000 },
+      build: { binRoot },
+    });
+
+    expect(outcome.kind).toBe("recorded");
+    if (outcome.kind !== "recorded") return;
+
+    const samples = store.read({ arm: "gitea-mcp", taskId: READ_TASK.id });
+    expect(samples).toHaveLength(1);
+    const [sample] = samples;
+    expect(sample).toBeDefined();
+    if (sample === undefined) return;
+
+    // A completed read cell: the checker scored the final report a pass.
+    expect(sample.outcome).toEqual({ pass: true });
+    // The record carries the agent's final report verbatim.
+    expect(sample.report).toBe(FINAL_REPORT);
+  });
+
+  // Behavior: a completed mutation cell produces a record WITHOUT an agent report
+  // (benchmark-harness spec, record-assembly seam). A mutation task is scored by
+  // diffing repository state, not by reading the agent's words, so the recorded
+  // ResultRecord must carry no `report` field at all — absent, not present-but-
+  // undefined. SAMPLE_TASK is a mutation task; paired with the passing fake host
+  // and driver it drives a completed mutation PASS. The absence is the independent
+  // source of truth (the acceptance criterion), read back through the store after
+  // the JSON round-trip so an absent field is genuinely not a key.
+  it("records no agent report on a completed mutation cell", async () => {
+    const { host } = createFakeHost();
+    const store = createSampleStore(storeRoot);
+
+    const outcome = await runCell({
+      arm: "gitea-mcp",
+      task: SAMPLE_TASK,
+      trial: 1,
+      access: ACCESS,
+      host,
+      driver,
+      store,
+      bounds: { turnCap: 10, wallClockMs: 60_000 },
+      build: { binRoot },
+    });
+
+    expect(outcome.kind).toBe("recorded");
+    if (outcome.kind !== "recorded") return;
+
+    const samples = store.read({ arm: "gitea-mcp", taskId: SAMPLE_TASK.id });
+    expect(samples).toHaveLength(1);
+    const [sample] = samples;
+    expect(sample).toBeDefined();
+    if (sample === undefined) return;
+
+    // A completed mutation cell: the state diff scored a pass.
+    expect(sample.outcome).toEqual({ pass: true });
+    // A mutation record carries no agent report — the key is absent entirely.
+    expect(sample).not.toHaveProperty("report");
+  });
 });

bench/runner.ts

@@ -15,7 +15,7 @@ import { buildArm, type ArmDefinition, type BuildArmOptions, type SharedContext
 import { auditTranscript, type ToolUse } from "./audit.js";
 import { score } from "./checker.js";
 import type { Arm, Outcome, ResultRecord, TokenComponents } from "./result.js";
-import type { RepoState } from "./scoring-spec.js";
+import type { RepoState, ScoringSpec } from "./scoring-spec.js";
 import type { BenchAccess, RepoCoords } from "./seed.js";
 import type { SampleStore } from "./store.js";
 import type { BenchTask } from "./task.js";
@@ -149,7 +149,10 @@ export async function runCell(input: RunCellInput): Promise<CellOutcome> {
     // A hung run produced no completed transcript to audit or score; record it
     // as a failure with no measured consumption.
     if (result.kind === "hung") {
-      return recorded(store, makeRecord(input, NO_TOKENS, 0, 0, durationMs, { pass: false, failure: "hung" }, clock));
+      return recorded(
+        store,
+        makeRecord(input, NO_TOKENS, 0, 0, durationMs, { pass: false, failure: "hung" }, undefined, clock),
+      );
     }
 
     const run = result.run;
@@ -161,13 +164,19 @@ export async function runCell(input: RunCellInput): Promise<CellOutcome> {
       return { kind: "invalid", leaks: audit.leaks };
     }
 
+    const spec = task.scoringSpec(coords.owner);
     const outcome = run.stoppedByTurnCap
       ? ({ pass: false, failure: "confused" } as const)
-      : await scoreRun(host, coords, task, run);
+      : await scoreRun(host, coords, spec, run);
+
+    // Retain the agent's final report for read tasks so a failed read is
+    // diagnosable directly from the record; mutation tasks are scored by diffing
+    // repository state and have no agent report to record.
+    const report = spec.kind === "read" ? run.finalReport : undefined;
 
     return recorded(
       store,
-      makeRecord(input, run.tokens, run.turns, run.imputedCostUsd, durationMs, outcome, clock),
+      makeRecord(input, run.tokens, run.turns, run.imputedCostUsd, durationMs, outcome, report, clock),
     );
   } finally {
     await host.delete(coords);
@@ -214,8 +223,7 @@ async function runBounded(
  * final report against the required facts for a read. A pass is a pass; anything
  * the checker rejects is an incorrect failure.
  */
-async function scoreRun(host: BenchHost, coords: RepoCoords, task: BenchTask, run: AgentRun): Promise<Outcome> {
-  const spec = task.scoringSpec(coords.owner);
+async function scoreRun(host: BenchHost, coords: RepoCoords, spec: ScoringSpec, run: AgentRun): Promise<Outcome> {
   const snapshot = await host.capture(coords);
   const check =
     spec.kind === "mutation"
@@ -232,6 +240,7 @@ function makeRecord(
   imputedCostUsd: number,
   durationMs: number,
   outcome: Outcome,
+  report: string | undefined,
   clock: RunnerClock,
 ): ResultRecord {
   return {
@@ -245,6 +254,9 @@ function makeRecord(
     durationMs,
     imputedCostUsd,
     outcome,
+    // Absent for mutation runs and runs with no completed report (hung); JSON
+    // serialization drops the key when undefined.
+    ...(report !== undefined ? { report } : {}),
   };
 }
 

bench/store.test.ts

@@ -102,4 +102,24 @@ describe("SampleStore", () => {
     secondRun.append(second);
     expect(secondRun.read(cell)).toEqual([first, second]);
   });
+
+  // Behavior: the store round-trips a report-bearing record without any change to
+  // the store itself. A read task's record carries the agent's final report in the
+  // `report` field; appending it and reading it back must return it equal, `report`
+  // and all, proving the store persists the field with no store change. The planted
+  // report string is an independent literal, not anything production code computes.
+  it("round-trips a record carrying a report field, preserving it unchanged", () => {
+    const store = createSampleStore(root);
+    const report = "There are 5 open issues in the repository.";
+    const record = sample({ tier: "read", report });
+
+    store.append(record);
+
+    const readBack = store.read({ arm: record.arm, taskId: record.taskId });
+    expect(readBack).toEqual([record]);
+    const [only] = readBack;
+    expect(only).toBeDefined();
+    if (only === undefined) return;
+    expect(only.report).toBe(report);
+  });
 });