From 14d494afa2dc6ccda8e5f9aa81e91f69b748f0dc Mon Sep 17 00:00:00 2001 From: alexion Date: Fri, 17 Jul 2026 14:43:43 -0400 Subject: [PATCH] feat: persist the tool transcript on every result record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records stored only a run's token/turn totals, so an arm's turn cost — the dominant driver of cache-read tokens — could not be diagnosed from the store. Retain the ordered transcript of tool invocations (the exact shell commands, MCP calls, and built-in tools the run made) on every scored record, absent only for a hung run that produced no transcript. The canonical TranscriptEntry shape lives on the record (result.ts); the isolation audit's ToolUse now aliases it so the persisted and audited shapes cannot drift. --- bench/audit.ts | 10 +++++----- bench/result.ts | 22 ++++++++++++++++++++++ bench/runner.test.ts | 6 ++++++ bench/runner.ts | 10 +++++++--- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/bench/audit.ts b/bench/audit.ts index c2e86f1..7e28876 100644 --- a/bench/audit.ts +++ b/bench/audit.ts @@ -12,17 +12,17 @@ // runner (runner.ts) drives the run and feeds the transcript here. import type { ArmDefinition } from "./arm.js"; +import type { TranscriptEntry } from "./result.js"; /** * One tool invocation recorded in the agent's transcript, reduced to what the * isolation audit needs. `shell` is a proposed shell command; `mcp` is a call to * an attached MCP server's tool; `other` is a built-in, non-Gitea-reaching tool - * (file read/edit and the like) that carries no isolation risk. + * (file read/edit and the like) that carries no isolation risk. This is the same + * shape the record persists ({@link TranscriptEntry}); the audit and the record + * share one type so they cannot drift. */ -export type ToolUse = - | { kind: "shell"; command: string } - | { kind: "mcp"; server: string; tool: string } - | { kind: "other"; name: string }; +export type ToolUse = TranscriptEntry; /** * The audit's verdict. On a leak it carries a human-readable reason per foreign diff --git a/bench/result.ts b/bench/result.ts index 6327320..9f0639d 100644 --- a/bench/result.ts +++ b/bench/result.ts @@ -45,6 +45,19 @@ export type FailureTag = "incorrect" | "confused" | "hung"; /** The pass/fail outcome of a run, tagged with the failure mode when it fails. */ export type Outcome = { pass: true } | { pass: false; failure: FailureTag }; +/** + * One tool invocation as it is recorded in a run's transcript, in the order it + * executed. This is the canonical shape the harness both audits for isolation + * (see audit.ts, whose `ToolUse` aliases this) and persists on the record for + * diagnosis. A `shell` entry keeps the exact command line the agent ran; an `mcp` + * entry names the server and tool it called; `other` names a built-in tool that + * reaches no Gitea channel. + */ +export type TranscriptEntry = + | { kind: "shell"; command: string } + | { kind: "mcp"; server: string; tool: string } + | { kind: "other"; name: string }; + /** * One completed `(arm, task, trial)` run. Carries the metrics the headline and * supporting views are computed from, plus the tags those views group by. @@ -81,6 +94,15 @@ export interface ResultRecord { * scored by diffing repository state and have no agent report to record. */ report?: string; + + /** + * The ordered transcript of tool invocations the run made, retained on every + * scored run so its turn cost is diagnosable directly from the record — the + * exact command sequence, which is how an arm's turn count (the dominant driver + * of cache-read tokens) is explained. Absent only for a hung run, which + * produced no completed transcript to record. + */ + transcript?: TranscriptEntry[]; } /** diff --git a/bench/runner.test.ts b/bench/runner.test.ts index 4747354..f4cda06 100644 --- a/bench/runner.test.ts +++ b/bench/runner.test.ts @@ -241,6 +241,12 @@ describe("runCell", () => { expect(sample.imputedCostUsd).toBe(DRIVER_COST); expect(sample.outcome).toEqual({ pass: true }); + // The recorded sample carries the run's tool transcript — the exact ordered + // sequence of tool invocations the driver reported — so the turn's cost is + // diagnosable directly from the record. The expected value is the literal the + // fake driver planted, deep-equal and in order, not recomputed from runner.ts. + expect(sample.transcript).toEqual([{ kind: "mcp", server: "gitea-mcp", tool: "edit_issue" }]); + // The sample carries the run's wall-clock duration; a completed run takes // non-negative time. expect(typeof sample.durationMs).toBe("number"); diff --git a/bench/runner.ts b/bench/runner.ts index 25caa3f..3e7ff81 100644 --- a/bench/runner.ts +++ b/bench/runner.ts @@ -14,7 +14,7 @@ import { buildArm, type ArmDefinition, type BuildArmOptions, type SharedContext } from "./arm.js"; import { auditTranscript, type ToolUse } from "./audit.js"; import { score } from "./checker.js"; -import type { Arm, Outcome, ResultRecord, TokenComponents } from "./result.js"; +import type { Arm, Outcome, ResultRecord, TokenComponents, TranscriptEntry } from "./result.js"; import type { RepoState, ScoringSpec } from "./scoring-spec.js"; import type { BenchAccess, RepoCoords } from "./seed.js"; import type { SampleStore } from "./store.js"; @@ -151,7 +151,7 @@ export async function runCell(input: RunCellInput): Promise { if (result.kind === "hung") { return recorded( store, - makeRecord(input, NO_TOKENS, 0, 0, durationMs, { pass: false, failure: "hung" }, undefined, clock), + makeRecord(input, NO_TOKENS, 0, 0, durationMs, { pass: false, failure: "hung" }, undefined, undefined, clock), ); } @@ -176,7 +176,7 @@ export async function runCell(input: RunCellInput): Promise { return recorded( store, - makeRecord(input, run.tokens, run.turns, run.imputedCostUsd, durationMs, outcome, report, clock), + makeRecord(input, run.tokens, run.turns, run.imputedCostUsd, durationMs, outcome, report, run.transcript, clock), ); } finally { await host.delete(coords); @@ -241,6 +241,7 @@ function makeRecord( durationMs: number, outcome: Outcome, report: string | undefined, + transcript: TranscriptEntry[] | undefined, clock: RunnerClock, ): ResultRecord { return { @@ -257,6 +258,9 @@ function makeRecord( // Absent for mutation runs and runs with no completed report (hung); JSON // serialization drops the key when undefined. ...(report !== undefined ? { report } : {}), + // Absent only for a hung run, which produced no transcript; JSON + // serialization drops the key when undefined. + ...(transcript !== undefined ? { transcript } : {}), }; }