From 9bf8c85dc34f84809042134a1d8f7cd12a52be4e Mon Sep 17 00:00:00 2001 From: alexion Date: Wed, 15 Jul 2026 10:20:15 -0400 Subject: [PATCH] feat: add benchmark scaffold and result store (task 0022) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lay the foundation the benchmark harness reads and writes: a bench/ directory (excluded from the published npm package), the immutable result-record shape, and an append-only per-cell sample store. - bench/result.ts: the ResultRecord shape — four token components, turns, duration, imputed cost, tagged pass/fail outcome, and the arm/task/tier/trial/timestamp tags. Arm and Tier are typed unions. - bench/store.ts: append-only sample store, one JSONL file per cell at //.jsonl; deepening a cell only ever adds samples. - bench/README.md: harness working docs and benchmark vocabulary, kept out of the tool's domain glossary per the spec. - Dedicated bench test tier (vitest.bench.config.ts, npm run test:bench) kept out of the fast tier; tsconfig typechecks bench. - Packaging tier asserts bench/ never ships in the tarball. --- .../0022-bench-scaffold-and-result-store.md | 17 ++- bench/README.md | 47 ++++++++ bench/result.ts | 84 ++++++++++++++ bench/store.test.ts | 105 ++++++++++++++++++ bench/store.ts | 86 ++++++++++++++ package.json | 3 +- test/packaging/packaging.test.ts | 4 + tsconfig.json | 2 +- vitest.bench.config.ts | 13 +++ 9 files changed, 355 insertions(+), 6 deletions(-) create mode 100644 bench/README.md create mode 100644 bench/result.ts create mode 100644 bench/store.test.ts create mode 100644 bench/store.ts create mode 100644 vitest.bench.config.ts diff --git a/.claude/tasks/0022-bench-scaffold-and-result-store.md b/.claude/tasks/0022-bench-scaffold-and-result-store.md index 05b06eb..bcfb34f 100644 --- a/.claude/tasks/0022-bench-scaffold-and-result-store.md +++ b/.claude/tasks/0022-bench-scaffold-and-result-store.md @@ -12,7 +12,16 @@ The store appends records as immutable, timestamped samples to a per-cell locati ## Acceptance criteria -- [ ] A `bench/` directory exists and is excluded from the npm package (verified by the packaging tier or an equivalent `files` check). -- [ ] The result-record shape records the four token components, turns, duration, imputed cost, outcome, failure tag, and the arm/task/tier/trial/timestamp tags. -- [ ] Appending a sample to a cell that already has samples leaves the prior samples intact; reading the cell returns all of them. -- [ ] A round-trip test writes several samples across cells and reads back exactly what was written. +- [x] A `bench/` directory exists and is excluded from the npm package (verified by the packaging tier or an equivalent `files` check). +- [x] The result-record shape records the four token components, turns, duration, imputed cost, outcome, failure tag, and the arm/task/tier/trial/timestamp tags. +- [x] Appending a sample to a cell that already has samples leaves the prior samples intact; reading the cell returns all of them. +- [x] A round-trip test writes several samples across cells and reads back exactly what was written. + +## Implementation Notes + +- The harness lives in `bench/`, kept out of the published package by the existing `files` allow-list (`["dist", "skills"]`); a new assertion in the packaging tier (`test/packaging/packaging.test.ts`) locks in that `package/bench` never ships. +- `bench/result.ts` holds the immutable `ResultRecord` shape. `Arm` and `Tier` are typed unions (the four arms; the four task tiers from the spec) rather than bare strings, and the failure tag is `"incorrect" | "confused" | "hung"` — the spec/task only required distinguishing confused (turn cap) from hung (wall-clock backstop), and `incorrect` is added for the ordinary checker-scored-wrong failure the later runner will record. +- `bench/store.ts` is the append-only sample store, backed by one newline-delimited JSON file per cell at `//.jsonl`. Immutability is structural: the store exposes only `append`/`read`/`cells`, and append is a bare file append, so deepening a cell can only add lines. `cells()` enumerates written cells — slightly beyond the literal criteria but foundational for the aggregator slice (0030), which reads the store. +- Bench tests run in a dedicated tier (`vitest.bench.config.ts`, `npm run test:bench`), colocated with the source, kept out of the fast tier so harness code never counts against the `src/` coverage thresholds. `tsconfig.json` now includes `bench` so the harness is typechecked. +- Benchmark vocabulary (arm, cell, tier, cost-equivalent tokens, seed, checker) is documented in `bench/README.md`, deliberately kept out of the tool's domain glossary (`.claude/CONTEXT.md`) per the spec's Further Notes. +- Review: Risk overall Low; Spec axis clean; two Standards judgement-call Duplicated Code findings addressed in the refactor — `append` now routes through `cellPath`, and the repeated ENOENT handling is factored into one `ignoreEnoent` helper. diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..61b87b0 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,47 @@ +# Benchmark harness + +This directory holds the benchmark harness that measures gitea-axi's central claim — that it is an agent-ergonomic, low-token interface to Gitea — against the `tea` CLI, the official `gitea-mcp` server, and raw Gitea REST calls. + +It is **not part of the published npm package**. +The package's `files` allow-list ships only `dist` and `skills`; `bench/` is excluded, and the packaging tier asserts it stays out of the tarball. + +The design lives in [`.claude/spec/benchmark-harness.md`](../.claude/spec/benchmark-harness.md) and the `.claude/adr/0014`–`0016` decision records. +This README is the harness's own working documentation; it deliberately keeps the benchmark's vocabulary here rather than in the tool's domain glossary ([`.claude/CONTEXT.md`](../.claude/CONTEXT.md)), which describes gitea-axi's own language. + +## Vocabulary + +**arm** — one of the four tool conditions under comparison: `gitea-axi`, `tea`, `gitea-mcp`, `raw-api`. +The comparison measures the tool, so the agent in each arm is given exactly one arm's tool. + +**cell** — one `(arm, task)` pair. +A cell's trials accumulate as samples within it; deepening a cell's sample size adds samples rather than overwriting prior runs. + +**trial** — one run of a cell. +Each cell defaults to five trials with a reporting floor of three. + +**tier** — the task category a task belongs to: `read`, `single-mutation`, `find-then-act`, `multi-step`. +Views group by tier to show where an arm wins or loses. + +**cost-equivalent tokens** — the headline metric: the four token components weighted by Anthropic's published API pricing ratios (see ADR 0014). +The raw component breakdown is retained on every sample so the data can be re-weighted without re-running. + +**seed** — the deterministic, idempotent starting state scripted into each throwaway repository before a trial, against which correctness is scored. + +**checker** — the deterministic scorer that diffs post-run repository state (mutation tasks) or matches required facts in the agent's report (read tasks) against the seeded ground truth. + +## Layout + +- `result.ts` — the immutable result-record shape and its tags (arm, task, tier, trial, timestamp). +- `store.ts` — the append-only, per-cell sample store that accumulates result records. + +Later slices add the tool-isolation guard, the seed provisioning, the arm scaffolding, the single-cell runner, the task suite, the run-loop CLI, and the aggregator. + +## Tests + +The harness's deterministic seams are unit-tested in this directory, colocated with their source, and run via: + +``` +npm run test:bench +``` + +They are kept out of the main fast tier so harness code never counts against the `src/` coverage thresholds. diff --git a/bench/result.ts b/bench/result.ts new file mode 100644 index 0000000..323678e --- /dev/null +++ b/bench/result.ts @@ -0,0 +1,84 @@ +// The immutable result-record shape that the whole benchmark harness reads and +// writes. One record captures one completed `(arm, task, trial)` run. Records +// are never mutated after they are written; deepening a cell's sample size +// appends new records rather than overwriting prior ones (see store.ts). +// +// The benchmark's own vocabulary (arm, cell, tier, cost-equivalent tokens) is +// documented in bench/README.md and the benchmark-harness spec, deliberately +// kept out of the tool's own domain glossary. + +/** The four tool conditions the benchmark compares. */ +export type Arm = "gitea-axi" | "tea" | "gitea-mcp" | "raw-api"; + +/** + * The task tiers the scored suite is weighted across. Views group by tier to + * show where an arm wins or loses. + */ +export type Tier = "read" | "single-mutation" | "find-then-act" | "multi-step"; + +/** + * The four token components retained per run. They are kept separate (rather + * than pre-summed) so cost-equivalent tokens can be re-weighted at render time + * without re-running — see the cost-equivalent-token-metric ADR. The auxiliary + * small model the runtime invokes for internal chores is folded into these + * counts, because it is real consumption against the same allowance. + */ +export interface TokenComponents { + /** Fresh, uncached input tokens (weighted 1x). */ + freshInput: number; + /** Cache-creation (write) tokens. */ + cacheCreation: number; + /** Cache-read tokens. */ + cacheRead: number; + /** Output tokens. */ + output: number; +} + +/** + * Why a run failed. `incorrect` means the agent finished but the checker scored + * the outcome wrong; `confused` means it hit the turn cap; `hung` means it hit + * the wall-clock backstop. The confused-versus-hung split lets the reporting + * distinguish a lost agent from a stuck one. + */ +export type FailureTag = "incorrect" | "confused" | "hung"; + +/** The pass/fail outcome of a run, tagged with the failure mode when it fails. */ +export type Outcome = { pass: true } | { pass: false; failure: FailureTag }; + +/** + * One completed `(arm, task, trial)` run. Carries the metrics the headline and + * supporting views are computed from, plus the tags those views group by. + */ +export interface ResultRecord { + /** The arm under test. */ + arm: Arm; + /** The task's stable identifier. */ + taskId: string; + /** The task's tier. */ + tier: Tier; + /** The trial index within the cell (1-based). */ + trial: number; + /** ISO 8601 timestamp of when the sample was recorded. */ + timestamp: string; + + /** The four token components. */ + tokens: TokenComponents; + /** Number of agent turns the run took. */ + turns: number; + /** Wall-clock duration in milliseconds. */ + durationMs: number; + /** The runtime's imputed cost in US dollars, retained as a secondary metric. */ + imputedCostUsd: number; + + /** Pass/fail outcome with a failure tag. */ + outcome: Outcome; +} + +/** + * The address of a cell in the sample store. A cell is one `(arm, task)` pair; + * its trials accumulate as samples within it. + */ +export interface CellKey { + arm: Arm; + taskId: string; +} diff --git a/bench/store.test.ts b/bench/store.test.ts new file mode 100644 index 0000000..d6e6ada --- /dev/null +++ b/bench/store.test.ts @@ -0,0 +1,105 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { ResultRecord } from "./result.js"; +import { createSampleStore } from "./store.js"; + +/** Build a valid ResultRecord with sensible defaults, overridable per test. */ +function sample(overrides: Partial = {}): ResultRecord { + return { + arm: "gitea-axi", + taskId: "issue-triage", + tier: "single-mutation", + trial: 1, + timestamp: "2026-07-15T12:00:00Z", + tokens: { freshInput: 100, cacheCreation: 200, cacheRead: 300, output: 40 }, + turns: 5, + durationMs: 1234, + imputedCostUsd: 0.0123, + outcome: { pass: true }, + ...overrides, + }; +} + +describe("SampleStore", () => { + let root: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "gitea-axi-store-")); + }); + + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + it("reads back a sample appended to a cell", () => { + const store = createSampleStore(root); + const record = sample(); + + store.append(record); + + expect(store.read({ arm: record.arm, taskId: record.taskId })).toEqual([record]); + }); + + it("preserves prior samples when appending to a cell, returning all in append order", () => { + const store = createSampleStore(root); + const first = sample({ trial: 1, outcome: { pass: true } }); + const second = sample({ trial: 2, outcome: { pass: false, failure: "incorrect" } }); + + store.append(first); + store.append(second); + + expect(store.read({ arm: "gitea-axi", taskId: "issue-triage" })).toEqual([first, second]); + }); + + it("keeps cells isolated on read and enumerates every written cell via cells()", () => { + const store = createSampleStore(root); + + // Cell A: two samples (gitea-axi / issue-triage, read tier). + const a1 = sample({ arm: "gitea-axi", taskId: "issue-triage", tier: "read", trial: 1 }); + const a2 = sample({ arm: "gitea-axi", taskId: "issue-triage", tier: "read", trial: 2 }); + // Cell B: one sample (tea / pr-review, single-mutation tier). + const b1 = sample({ arm: "tea", taskId: "pr-review", tier: "single-mutation", trial: 1 }); + // Cell C: one sample (gitea-mcp / label-sync, multi-step tier). + const c1 = sample({ arm: "gitea-mcp", taskId: "label-sync", tier: "multi-step", trial: 1 }); + + // Interleave appends across cells to exercise isolation of the write path. + store.append(a1); + store.append(b1); + store.append(a2); + store.append(c1); + + expect(store.read({ arm: "gitea-axi", taskId: "issue-triage" })).toEqual([a1, a2]); + expect(store.read({ arm: "tea", taskId: "pr-review" })).toEqual([b1]); + expect(store.read({ arm: "gitea-mcp", taskId: "label-sync" })).toEqual([c1]); + + const enumerated = store.cells(); + expect(enumerated).toHaveLength(3); + expect(enumerated).toEqual( + expect.arrayContaining([ + { arm: "gitea-axi", taskId: "issue-triage" }, + { arm: "tea", taskId: "pr-review" }, + { arm: "gitea-mcp", taskId: "label-sync" }, + ]), + ); + }); + + it("reads back accumulated samples through a fresh store on the same root, deepening across runs", () => { + const cell = { arm: "gitea-axi", taskId: "issue-triage" } as const; + const first = sample({ trial: 1, outcome: { pass: true } }); + const second = sample({ trial: 2, outcome: { pass: false, failure: "hung" } }); + + // First "run": append one sample, then let this store handle go out of scope. + const firstRun = createSampleStore(root); + firstRun.append(first); + + // Second, independent "run" on the same root — a later process reopening it. + const secondRun = createSampleStore(root); + expect(secondRun.read(cell)).toEqual([first]); + + // Deepening the cell through the second store accumulates rather than resets. + secondRun.append(second); + expect(secondRun.read(cell)).toEqual([first, second]); + }); +}); diff --git a/bench/store.ts b/bench/store.ts new file mode 100644 index 0000000..cfea61a --- /dev/null +++ b/bench/store.ts @@ -0,0 +1,86 @@ +import { appendFileSync, mkdirSync, readdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import type { CellKey, ResultRecord } from "./result.js"; + +/** Run `read`, returning `fallback` when the target does not exist yet. */ +function ignoreEnoent(read: () => T, fallback: T): T { + try { + return read(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return fallback; + } + throw error; + } +} + +/** + * An append-only store of result samples, one location per cell. Deepening a + * cell's sample size appends new records; prior samples are never overwritten, + * and reading a cell returns every accumulated sample. + */ +export interface SampleStore { + /** Append one immutable sample to the cell derived from its arm and task id. */ + append(record: ResultRecord): void; + /** Read every sample accumulated for a cell, in append order; `[]` if none. */ + read(cell: CellKey): ResultRecord[]; + /** Enumerate every cell that has at least one sample. */ + cells(): CellKey[]; +} + +/** + * Open a sample store rooted at `root`. The store is backed by the filesystem so + * accumulated samples persist across processes and can be deepened over many + * separate runs. + * + * Each cell is one newline-delimited JSON file at `//.jsonl`. + * Append is a bare file append, which is what makes prior samples immutable: + * deepening a cell only ever adds lines. Reading parses the whole file back. + */ +export function createSampleStore(root: string): SampleStore { + function cellPath(cell: CellKey): string { + return join(root, cell.arm, `${cell.taskId}.jsonl`); + } + + return { + append(record) { + const path = cellPath({ arm: record.arm, taskId: record.taskId }); + mkdirSync(dirname(path), { recursive: true }); + appendFileSync(path, `${JSON.stringify(record)}\n`); + }, + + read(cell) { + const contents = ignoreEnoent( + () => readFileSync(cellPath(cell), "utf8"), + undefined, + ); + if (contents === undefined) { + return []; + } + return contents + .split("\n") + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as ResultRecord); + }, + + cells() { + const found: CellKey[] = []; + const arms = ignoreEnoent(() => readdirSync(root), []); + for (const arm of arms) { + let files: string[]; + try { + files = readdirSync(join(root, arm)); + } catch { + // Not a directory (a stray file at the root): no cells live under it. + continue; + } + for (const file of files) { + if (file.endsWith(".jsonl")) { + found.push({ arm: arm as CellKey["arm"], taskId: file.slice(0, -".jsonl".length) }); + } + } + } + return found; + }, + }; +} diff --git a/package.json b/package.json index 820a3ad..15d7790 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:e2e": "vitest run --config vitest.e2e.config.ts", - "test:pack": "vitest run --config vitest.packaging.config.ts" + "test:pack": "vitest run --config vitest.packaging.config.ts", + "test:bench": "vitest run --config vitest.bench.config.ts" }, "dependencies": { "@toon-format/toon": "^2.3.0", diff --git a/test/packaging/packaging.test.ts b/test/packaging/packaging.test.ts index de48ea4..281b721 100644 --- a/test/packaging/packaging.test.ts +++ b/test/packaging/packaging.test.ts @@ -103,6 +103,10 @@ describe("npm distribution artifact", () => { expect(scripts?.postinstall).toBeUndefined(); }); + it("excludes the bench/ harness directory from the package", () => { + expect(existsSync(join(extractDir, "package", "bench"))).toBe(false); + }); + it("declares complete metadata: unscoped name, description, repo, license, engines, ESM type", () => { const name = packedManifest.name as string; expect(name).toBe("gitea-axi"); diff --git a/tsconfig.json b/tsconfig.json index b3612e5..d1f9b9f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,5 +11,5 @@ "types": ["node"], "noEmit": true }, - "include": ["src", "test", "vitest.config.ts"] + "include": ["src", "test", "bench", "vitest.config.ts", "vitest.bench.config.ts"] } diff --git a/vitest.bench.config.ts b/vitest.bench.config.ts new file mode 100644 index 0000000..11abfeb --- /dev/null +++ b/vitest.bench.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + // The benchmark harness tier. The harness lives in bench/ (excluded from the + // published npm package) and its deterministic seams — the sample store, + // checker, guard, and aggregator — are unit-tested here, colocated with the + // source. It runs on its own via `test:bench` and is kept out of the main + // fast tier so bench code never counts against src coverage thresholds. + include: ["bench/**/*.test.ts"], + passWithNoTests: true, + }, +});