bench/audit.ts

@@ -12,17 +12,17 @@
 // runner (runner.ts) drives the run and feeds the transcript here.
 
 import type { ArmDefinition } from "./arm.js";
+import type { TranscriptEntry } from "./result.js";
 
 /**
  * One tool invocation recorded in the agent's transcript, reduced to what the
  * isolation audit needs. `shell` is a proposed shell command; `mcp` is a call to
  * an attached MCP server's tool; `other` is a built-in, non-Gitea-reaching tool
- * (file read/edit and the like) that carries no isolation risk.
+ * (file read/edit and the like) that carries no isolation risk. This is the same
+ * shape the record persists ({@link TranscriptEntry}); the audit and the record
+ * share one type so they cannot drift.
  */
-export type ToolUse =
+export type ToolUse = TranscriptEntry;
-  | { kind: "shell"; command: string }
-  | { kind: "mcp"; server: string; tool: string }
-  | { kind: "other"; name: string };
 
 /**
  * The audit's verdict. On a leak it carries a human-readable reason per foreign

bench/result.ts

@@ -45,6 +45,19 @@ export type FailureTag = "incorrect" | "confused" | "hung";
 /** The pass/fail outcome of a run, tagged with the failure mode when it fails. */
 export type Outcome = { pass: true } | { pass: false; failure: FailureTag };
 
+/**
+ * One tool invocation as it is recorded in a run's transcript, in the order it
+ * executed. This is the canonical shape the harness both audits for isolation
+ * (see audit.ts, whose `ToolUse` aliases this) and persists on the record for
+ * diagnosis. A `shell` entry keeps the exact command line the agent ran; an `mcp`
+ * entry names the server and tool it called; `other` names a built-in tool that
+ * reaches no Gitea channel.
+ */
+export type TranscriptEntry =
+  | { kind: "shell"; command: string }
+  | { kind: "mcp"; server: string; tool: string }
+  | { kind: "other"; name: string };
+
 /**
  * One completed `(arm, task, trial)` run. Carries the metrics the headline and
  * supporting views are computed from, plus the tags those views group by.
@@ -81,6 +94,15 @@ export interface ResultRecord {
    * scored by diffing repository state and have no agent report to record.
    */
   report?: string;
+
+  /**
+   * The ordered transcript of tool invocations the run made, retained on every
+   * scored run so its turn cost is diagnosable directly from the record — the
+   * exact command sequence, which is how an arm's turn count (the dominant driver
+   * of cache-read tokens) is explained. Absent only for a hung run, which
+   * produced no completed transcript to record.
+   */
+  transcript?: TranscriptEntry[];
 }
 
 /**

bench/runner.test.ts

@@ -241,6 +241,12 @@ describe("runCell", () => {
     expect(sample.imputedCostUsd).toBe(DRIVER_COST);
     expect(sample.outcome).toEqual({ pass: true });
 
+    // The recorded sample carries the run's tool transcript — the exact ordered
+    // sequence of tool invocations the driver reported — so the turn's cost is
+    // diagnosable directly from the record. The expected value is the literal the
+    // fake driver planted, deep-equal and in order, not recomputed from runner.ts.
+    expect(sample.transcript).toEqual([{ kind: "mcp", server: "gitea-mcp", tool: "edit_issue" }]);
+
     // The sample carries the run's wall-clock duration; a completed run takes
     // non-negative time.
     expect(typeof sample.durationMs).toBe("number");

bench/runner.ts

@@ -14,7 +14,7 @@
 import { buildArm, type ArmDefinition, type BuildArmOptions, type SharedContext } from "./arm.js";
 import { auditTranscript, type ToolUse } from "./audit.js";
 import { score } from "./checker.js";
-import type { Arm, Outcome, ResultRecord, TokenComponents } from "./result.js";
+import type { Arm, Outcome, ResultRecord, TokenComponents, TranscriptEntry } from "./result.js";
 import type { RepoState, ScoringSpec } from "./scoring-spec.js";
 import type { BenchAccess, RepoCoords } from "./seed.js";
 import type { SampleStore } from "./store.js";
@@ -151,7 +151,7 @@ export async function runCell(input: RunCellInput): Promise<CellOutcome> {
     if (result.kind === "hung") {
       return recorded(
         store,
-        makeRecord(input, NO_TOKENS, 0, 0, durationMs, { pass: false, failure: "hung" }, undefined, clock),
+        makeRecord(input, NO_TOKENS, 0, 0, durationMs, { pass: false, failure: "hung" }, undefined, undefined, clock),
       );
     }
 
@@ -176,7 +176,7 @@ export async function runCell(input: RunCellInput): Promise<CellOutcome> {
 
     return recorded(
       store,
-      makeRecord(input, run.tokens, run.turns, run.imputedCostUsd, durationMs, outcome, report, clock),
+      makeRecord(input, run.tokens, run.turns, run.imputedCostUsd, durationMs, outcome, report, run.transcript, clock),
     );
   } finally {
     await host.delete(coords);
@@ -241,6 +241,7 @@ function makeRecord(
   durationMs: number,
   outcome: Outcome,
   report: string | undefined,
+  transcript: TranscriptEntry[] | undefined,
   clock: RunnerClock,
 ): ResultRecord {
   return {
@@ -257,6 +258,9 @@ function makeRecord(
     // Absent for mutation runs and runs with no completed report (hung); JSON
     // serialization drops the key when undefined.
     ...(report !== undefined ? { report } : {}),
+    // Absent only for a hung run, which produced no transcript; JSON
+    // serialization drops the key when undefined.
+    ...(transcript !== undefined ? { transcript } : {}),
   };
 }