alexion
9bf8c85dc3
Lay the foundation the benchmark harness reads and writes: a bench/ directory (excluded from the published npm package), the immutable result-record shape, and an append-only per-cell sample store. - bench/result.ts: the ResultRecord shape — four token components, turns, duration, imputed cost, tagged pass/fail outcome, and the arm/task/tier/trial/timestamp tags. Arm and Tier are typed unions. - bench/store.ts: append-only sample store, one JSONL file per cell at <root>/<arm>/<taskId>.jsonl; deepening a cell only ever adds samples. - bench/README.md: harness working docs and benchmark vocabulary, kept out of the tool's domain glossary per the spec. - Dedicated bench test tier (vitest.bench.config.ts, npm run test:bench) kept out of the fast tier; tsconfig typechecks bench. - Packaging tier asserts bench/ never ships in the tarball.
106 lines
3.9 KiB
TypeScript
106 lines
3.9 KiB
TypeScript
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
import type { ResultRecord } from "./result.js";
|
|
import { createSampleStore } from "./store.js";
|
|
|
|
/** Build a valid ResultRecord with sensible defaults, overridable per test. */
|
|
function sample(overrides: Partial<ResultRecord> = {}): ResultRecord {
|
|
return {
|
|
arm: "gitea-axi",
|
|
taskId: "issue-triage",
|
|
tier: "single-mutation",
|
|
trial: 1,
|
|
timestamp: "2026-07-15T12:00:00Z",
|
|
tokens: { freshInput: 100, cacheCreation: 200, cacheRead: 300, output: 40 },
|
|
turns: 5,
|
|
durationMs: 1234,
|
|
imputedCostUsd: 0.0123,
|
|
outcome: { pass: true },
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
describe("SampleStore", () => {
|
|
let root: string;
|
|
|
|
beforeEach(() => {
|
|
root = mkdtempSync(join(tmpdir(), "gitea-axi-store-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(root, { recursive: true, force: true });
|
|
});
|
|
|
|
it("reads back a sample appended to a cell", () => {
|
|
const store = createSampleStore(root);
|
|
const record = sample();
|
|
|
|
store.append(record);
|
|
|
|
expect(store.read({ arm: record.arm, taskId: record.taskId })).toEqual([record]);
|
|
});
|
|
|
|
it("preserves prior samples when appending to a cell, returning all in append order", () => {
|
|
const store = createSampleStore(root);
|
|
const first = sample({ trial: 1, outcome: { pass: true } });
|
|
const second = sample({ trial: 2, outcome: { pass: false, failure: "incorrect" } });
|
|
|
|
store.append(first);
|
|
store.append(second);
|
|
|
|
expect(store.read({ arm: "gitea-axi", taskId: "issue-triage" })).toEqual([first, second]);
|
|
});
|
|
|
|
it("keeps cells isolated on read and enumerates every written cell via cells()", () => {
|
|
const store = createSampleStore(root);
|
|
|
|
// Cell A: two samples (gitea-axi / issue-triage, read tier).
|
|
const a1 = sample({ arm: "gitea-axi", taskId: "issue-triage", tier: "read", trial: 1 });
|
|
const a2 = sample({ arm: "gitea-axi", taskId: "issue-triage", tier: "read", trial: 2 });
|
|
// Cell B: one sample (tea / pr-review, single-mutation tier).
|
|
const b1 = sample({ arm: "tea", taskId: "pr-review", tier: "single-mutation", trial: 1 });
|
|
// Cell C: one sample (gitea-mcp / label-sync, multi-step tier).
|
|
const c1 = sample({ arm: "gitea-mcp", taskId: "label-sync", tier: "multi-step", trial: 1 });
|
|
|
|
// Interleave appends across cells to exercise isolation of the write path.
|
|
store.append(a1);
|
|
store.append(b1);
|
|
store.append(a2);
|
|
store.append(c1);
|
|
|
|
expect(store.read({ arm: "gitea-axi", taskId: "issue-triage" })).toEqual([a1, a2]);
|
|
expect(store.read({ arm: "tea", taskId: "pr-review" })).toEqual([b1]);
|
|
expect(store.read({ arm: "gitea-mcp", taskId: "label-sync" })).toEqual([c1]);
|
|
|
|
const enumerated = store.cells();
|
|
expect(enumerated).toHaveLength(3);
|
|
expect(enumerated).toEqual(
|
|
expect.arrayContaining([
|
|
{ arm: "gitea-axi", taskId: "issue-triage" },
|
|
{ arm: "tea", taskId: "pr-review" },
|
|
{ arm: "gitea-mcp", taskId: "label-sync" },
|
|
]),
|
|
);
|
|
});
|
|
|
|
it("reads back accumulated samples through a fresh store on the same root, deepening across runs", () => {
|
|
const cell = { arm: "gitea-axi", taskId: "issue-triage" } as const;
|
|
const first = sample({ trial: 1, outcome: { pass: true } });
|
|
const second = sample({ trial: 2, outcome: { pass: false, failure: "hung" } });
|
|
|
|
// First "run": append one sample, then let this store handle go out of scope.
|
|
const firstRun = createSampleStore(root);
|
|
firstRun.append(first);
|
|
|
|
// Second, independent "run" on the same root — a later process reopening it.
|
|
const secondRun = createSampleStore(root);
|
|
expect(secondRun.read(cell)).toEqual([first]);
|
|
|
|
// Deepening the cell through the second store accumulates rather than resets.
|
|
secondRun.append(second);
|
|
expect(secondRun.read(cell)).toEqual([first, second]);
|
|
});
|
|
});
|