feat: add benchmark scaffold and result store (task 0022)
All checks were successful
CI / test (pull_request) Successful in 50s
CI / test (push) Successful in 50s

Lay the foundation the benchmark harness reads and writes: a bench/
directory (excluded from the published npm package), the immutable
result-record shape, and an append-only per-cell sample store.

- bench/result.ts: the ResultRecord shape — four token components,
  turns, duration, imputed cost, tagged pass/fail outcome, and the
  arm/task/tier/trial/timestamp tags. Arm and Tier are typed unions.
- bench/store.ts: append-only sample store, one JSONL file per cell at
  <root>/<arm>/<taskId>.jsonl; deepening a cell only ever adds samples.
- bench/README.md: harness working docs and benchmark vocabulary, kept
  out of the tool's domain glossary per the spec.
- Dedicated bench test tier (vitest.bench.config.ts, npm run test:bench)
  kept out of the fast tier; tsconfig typechecks bench.
- Packaging tier asserts bench/ never ships in the tarball.
This commit was merged in pull request #23.
This commit is contained in:
2026-07-15 10:20:15 -04:00
parent d445b2bd2b
commit 9bf8c85dc3
9 changed files with 355 additions and 6 deletions

86
bench/store.ts Normal file
View File

@@ -0,0 +1,86 @@
import { appendFileSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import type { CellKey, ResultRecord } from "./result.js";
/** Run `read`, returning `fallback` when the target does not exist yet. */
function ignoreEnoent<T>(read: () => T, fallback: T): T {
try {
return read();
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return fallback;
}
throw error;
}
}
/**
* An append-only store of result samples, one location per cell. Deepening a
* cell's sample size appends new records; prior samples are never overwritten,
* and reading a cell returns every accumulated sample.
*/
export interface SampleStore {
/** Append one immutable sample to the cell derived from its arm and task id. */
append(record: ResultRecord): void;
/** Read every sample accumulated for a cell, in append order; `[]` if none. */
read(cell: CellKey): ResultRecord[];
/** Enumerate every cell that has at least one sample. */
cells(): CellKey[];
}
/**
* Open a sample store rooted at `root`. The store is backed by the filesystem so
* accumulated samples persist across processes and can be deepened over many
* separate runs.
*
* Each cell is one newline-delimited JSON file at `<root>/<arm>/<taskId>.jsonl`.
* Append is a bare file append, which is what makes prior samples immutable:
* deepening a cell only ever adds lines. Reading parses the whole file back.
*/
export function createSampleStore(root: string): SampleStore {
function cellPath(cell: CellKey): string {
return join(root, cell.arm, `${cell.taskId}.jsonl`);
}
return {
append(record) {
const path = cellPath({ arm: record.arm, taskId: record.taskId });
mkdirSync(dirname(path), { recursive: true });
appendFileSync(path, `${JSON.stringify(record)}\n`);
},
read(cell) {
const contents = ignoreEnoent<string | undefined>(
() => readFileSync(cellPath(cell), "utf8"),
undefined,
);
if (contents === undefined) {
return [];
}
return contents
.split("\n")
.filter((line) => line.length > 0)
.map((line) => JSON.parse(line) as ResultRecord);
},
cells() {
const found: CellKey[] = [];
const arms = ignoreEnoent<string[]>(() => readdirSync(root), []);
for (const arm of arms) {
let files: string[];
try {
files = readdirSync(join(root, arm));
} catch {
// Not a directory (a stray file at the root): no cells live under it.
continue;
}
for (const file of files) {
if (file.endsWith(".jsonl")) {
found.push({ arm: arm as CellKey["arm"], taskId: file.slice(0, -".jsonl".length) });
}
}
}
return found;
},
};
}