alexion/gitea-axi
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: persist the tool transcript on every result record

Browse Source 
Records stored only a run's token/turn totals, so an arm's turn cost —
the dominant driver of cache-read tokens — could not be diagnosed from
the store. Retain the ordered transcript of tool invocations (the exact
shell commands, MCP calls, and built-in tools the run made) on every
scored record, absent only for a hung run that produced no transcript.

The canonical TranscriptEntry shape lives on the record (result.ts); the
isolation audit's ToolUse now aliases it so the persisted and audited
shapes cannot drift.
This commit is contained in:
alexion
2026-07-17 14:43:43 -04:00
parent 05b72f6987
commit 14d494afa2
4 changed files with 40 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bench/runner.ts
View File

@@ -14,7 +14,7 @@
import { buildArm, type ArmDefinition, type BuildArmOptions, type SharedContext } from "./arm.js";
import { auditTranscript, type ToolUse } from "./audit.js";
import { score } from "./checker.js";
import type { Arm, Outcome, ResultRecord, TokenComponents } from "./result.js";
import type { Arm, Outcome, ResultRecord, TokenComponents, TranscriptEntry } from "./result.js";
import type { RepoState, ScoringSpec } from "./scoring-spec.js";
import type { BenchAccess, RepoCoords } from "./seed.js";
import type { SampleStore } from "./store.js";
@@ -151,7 +151,7 @@ export async function runCell(input: RunCellInput): Promise<CellOutcome> {
if (result.kind === "hung") {
return recorded(
store,
makeRecord(input, NO_TOKENS, 0, 0, durationMs, { pass: false, failure: "hung" }, undefined, clock),
makeRecord(input, NO_TOKENS, 0, 0, durationMs, { pass: false, failure: "hung" }, undefined, undefined, clock),
);
}
@@ -176,7 +176,7 @@ export async function runCell(input: RunCellInput): Promise<CellOutcome> {
return recorded(
store,
makeRecord(input, run.tokens, run.turns, run.imputedCostUsd, durationMs, outcome, report, clock),
makeRecord(input, run.tokens, run.turns, run.imputedCostUsd, durationMs, outcome, report, run.transcript, clock),
);
} finally {
await host.delete(coords);
@@ -241,6 +241,7 @@ function makeRecord(
durationMs: number,
outcome: Outcome,
report: string | undefined,
transcript: TranscriptEntry[] | undefined,
clock: RunnerClock,
): ResultRecord {
return {
@@ -257,6 +258,9 @@ function makeRecord(
// Absent for mutation runs and runs with no completed report (hung); JSON
// serialization drops the key when undefined.
...(report !== undefined ? { report } : {}),
// Absent only for a hung run, which produced no transcript; JSON
// serialization drops the key when undefined.
...(transcript !== undefined ? { transcript } : {}),
};
}