alexion
9bf8c85dc3
Lay the foundation the benchmark harness reads and writes: a bench/ directory (excluded from the published npm package), the immutable result-record shape, and an append-only per-cell sample store. - bench/result.ts: the ResultRecord shape — four token components, turns, duration, imputed cost, tagged pass/fail outcome, and the arm/task/tier/trial/timestamp tags. Arm and Tier are typed unions. - bench/store.ts: append-only sample store, one JSONL file per cell at <root>/<arm>/<taskId>.jsonl; deepening a cell only ever adds samples. - bench/README.md: harness working docs and benchmark vocabulary, kept out of the tool's domain glossary per the spec. - Dedicated bench test tier (vitest.bench.config.ts, npm run test:bench) kept out of the fast tier; tsconfig typechecks bench. - Packaging tier asserts bench/ never ships in the tarball.
85 lines
3.0 KiB
TypeScript
85 lines
3.0 KiB
TypeScript
// The immutable result-record shape that the whole benchmark harness reads and
|
|
// writes. One record captures one completed `(arm, task, trial)` run. Records
|
|
// are never mutated after they are written; deepening a cell's sample size
|
|
// appends new records rather than overwriting prior ones (see store.ts).
|
|
//
|
|
// The benchmark's own vocabulary (arm, cell, tier, cost-equivalent tokens) is
|
|
// documented in bench/README.md and the benchmark-harness spec, deliberately
|
|
// kept out of the tool's own domain glossary.
|
|
|
|
/** The four tool conditions the benchmark compares. */
|
|
export type Arm = "gitea-axi" | "tea" | "gitea-mcp" | "raw-api";
|
|
|
|
/**
|
|
* The task tiers the scored suite is weighted across. Views group by tier to
|
|
* show where an arm wins or loses.
|
|
*/
|
|
export type Tier = "read" | "single-mutation" | "find-then-act" | "multi-step";
|
|
|
|
/**
|
|
* The four token components retained per run. They are kept separate (rather
|
|
* than pre-summed) so cost-equivalent tokens can be re-weighted at render time
|
|
* without re-running — see the cost-equivalent-token-metric ADR. The auxiliary
|
|
* small model the runtime invokes for internal chores is folded into these
|
|
* counts, because it is real consumption against the same allowance.
|
|
*/
|
|
export interface TokenComponents {
|
|
/** Fresh, uncached input tokens (weighted 1x). */
|
|
freshInput: number;
|
|
/** Cache-creation (write) tokens. */
|
|
cacheCreation: number;
|
|
/** Cache-read tokens. */
|
|
cacheRead: number;
|
|
/** Output tokens. */
|
|
output: number;
|
|
}
|
|
|
|
/**
|
|
* Why a run failed. `incorrect` means the agent finished but the checker scored
|
|
* the outcome wrong; `confused` means it hit the turn cap; `hung` means it hit
|
|
* the wall-clock backstop. The confused-versus-hung split lets the reporting
|
|
* distinguish a lost agent from a stuck one.
|
|
*/
|
|
export type FailureTag = "incorrect" | "confused" | "hung";
|
|
|
|
/** The pass/fail outcome of a run, tagged with the failure mode when it fails. */
|
|
export type Outcome = { pass: true } | { pass: false; failure: FailureTag };
|
|
|
|
/**
|
|
* One completed `(arm, task, trial)` run. Carries the metrics the headline and
|
|
* supporting views are computed from, plus the tags those views group by.
|
|
*/
|
|
export interface ResultRecord {
|
|
/** The arm under test. */
|
|
arm: Arm;
|
|
/** The task's stable identifier. */
|
|
taskId: string;
|
|
/** The task's tier. */
|
|
tier: Tier;
|
|
/** The trial index within the cell (1-based). */
|
|
trial: number;
|
|
/** ISO 8601 timestamp of when the sample was recorded. */
|
|
timestamp: string;
|
|
|
|
/** The four token components. */
|
|
tokens: TokenComponents;
|
|
/** Number of agent turns the run took. */
|
|
turns: number;
|
|
/** Wall-clock duration in milliseconds. */
|
|
durationMs: number;
|
|
/** The runtime's imputed cost in US dollars, retained as a secondary metric. */
|
|
imputedCostUsd: number;
|
|
|
|
/** Pass/fail outcome with a failure tag. */
|
|
outcome: Outcome;
|
|
}
|
|
|
|
/**
|
|
* The address of a cell in the sample store. A cell is one `(arm, task)` pair;
|
|
* its trials accumulate as samples within it.
|
|
*/
|
|
export interface CellKey {
|
|
arm: Arm;
|
|
taskId: string;
|
|
}
|