feat: add benchmark scaffold and result store (task 0022)
Lay the foundation the benchmark harness reads and writes: a bench/ directory (excluded from the published npm package), the immutable result-record shape, and an append-only per-cell sample store. - bench/result.ts: the ResultRecord shape — four token components, turns, duration, imputed cost, tagged pass/fail outcome, and the arm/task/tier/trial/timestamp tags. Arm and Tier are typed unions. - bench/store.ts: append-only sample store, one JSONL file per cell at <root>/<arm>/<taskId>.jsonl; deepening a cell only ever adds samples. - bench/README.md: harness working docs and benchmark vocabulary, kept out of the tool's domain glossary per the spec. - Dedicated bench test tier (vitest.bench.config.ts, npm run test:bench) kept out of the fast tier; tsconfig typechecks bench. - Packaging tier asserts bench/ never ships in the tarball.
This commit was merged in pull request #23.
This commit is contained in:
86
bench/store.ts
Normal file
86
bench/store.ts
Normal file
@@ -0,0 +1,86 @@
|
||||
import { appendFileSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import type { CellKey, ResultRecord } from "./result.js";
|
||||
|
||||
/** Run `read`, returning `fallback` when the target does not exist yet. */
|
||||
function ignoreEnoent<T>(read: () => T, fallback: T): T {
|
||||
try {
|
||||
return read();
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return fallback;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* An append-only store of result samples, one location per cell. Deepening a
|
||||
* cell's sample size appends new records; prior samples are never overwritten,
|
||||
* and reading a cell returns every accumulated sample.
|
||||
*/
|
||||
export interface SampleStore {
|
||||
/** Append one immutable sample to the cell derived from its arm and task id. */
|
||||
append(record: ResultRecord): void;
|
||||
/** Read every sample accumulated for a cell, in append order; `[]` if none. */
|
||||
read(cell: CellKey): ResultRecord[];
|
||||
/** Enumerate every cell that has at least one sample. */
|
||||
cells(): CellKey[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Open a sample store rooted at `root`. The store is backed by the filesystem so
|
||||
* accumulated samples persist across processes and can be deepened over many
|
||||
* separate runs.
|
||||
*
|
||||
* Each cell is one newline-delimited JSON file at `<root>/<arm>/<taskId>.jsonl`.
|
||||
* Append is a bare file append, which is what makes prior samples immutable:
|
||||
* deepening a cell only ever adds lines. Reading parses the whole file back.
|
||||
*/
|
||||
export function createSampleStore(root: string): SampleStore {
|
||||
function cellPath(cell: CellKey): string {
|
||||
return join(root, cell.arm, `${cell.taskId}.jsonl`);
|
||||
}
|
||||
|
||||
return {
|
||||
append(record) {
|
||||
const path = cellPath({ arm: record.arm, taskId: record.taskId });
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
appendFileSync(path, `${JSON.stringify(record)}\n`);
|
||||
},
|
||||
|
||||
read(cell) {
|
||||
const contents = ignoreEnoent<string | undefined>(
|
||||
() => readFileSync(cellPath(cell), "utf8"),
|
||||
undefined,
|
||||
);
|
||||
if (contents === undefined) {
|
||||
return [];
|
||||
}
|
||||
return contents
|
||||
.split("\n")
|
||||
.filter((line) => line.length > 0)
|
||||
.map((line) => JSON.parse(line) as ResultRecord);
|
||||
},
|
||||
|
||||
cells() {
|
||||
const found: CellKey[] = [];
|
||||
const arms = ignoreEnoent<string[]>(() => readdirSync(root), []);
|
||||
for (const arm of arms) {
|
||||
let files: string[];
|
||||
try {
|
||||
files = readdirSync(join(root, arm));
|
||||
} catch {
|
||||
// Not a directory (a stray file at the root): no cells live under it.
|
||||
continue;
|
||||
}
|
||||
for (const file of files) {
|
||||
if (file.endsWith(".jsonl")) {
|
||||
found.push({ arm: arm as CellKey["arm"], taskId: file.slice(0, -".jsonl".length) });
|
||||
}
|
||||
}
|
||||
}
|
||||
return found;
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user