feat: add benchmark aggregator and reporting (task 0030)
Add bench/aggregate.ts: the pure aggregator seam that rolls the accumulated sample store into a readable comparison. aggregate produces a headline table (one row per arm, cost-equivalent tokens as the headline plus raw tokens, turns, duration, success rate, coverage, and imputed cost as a de-emphasized secondary column), per-tier and per-token-component breakdowns, and the separate bonus table; renderReport renders it as stable text. Cost-equivalent tokens are weighted at render time from the four retained components per ADR 0014, and incomplete coverage is annotated rather than hidden. Unit-tested against synthetic append-only sample stores.
This commit was merged in pull request #31.
This commit is contained in:
@@ -49,8 +49,9 @@ The raw component breakdown is retained on every sample so the data can be re-we
|
||||
- `sdk-driver.ts` — `sdkAgentDriver`, the production `AgentDriver`: it runs one arm through the Claude Agent SDK on the maintainer's subscription, enforcing isolation in-band via the SDK's permission callback (the arm's guard on every Bash command; the shell disabled on the MCP arm) and reporting the four token components (folding in the auxiliary small model), the turn count, the imputed cost, the transcript, and the final report. The SDK is loaded through a computed dynamic import so it is an optional peer needed only for live runs.
|
||||
- `run-loop.ts` — `runCells`, the run loop: it runs one chosen `(arm, task)` cell for a batch of trials (defaulting to five, with a reporting floor of three) by driving `runCell` and the sample store, deciding only how many trials to run and at what trial numbers. Deepening a cell continues numbering past the highest trial it already holds and appends, so a cell's sample size grows across sittings rather than being overwritten. It reimplements no orchestration — provision, run, score, and append stay in `runCell`.
|
||||
- `run.ts` — the maintainer-facing run-loop command. `parseRunArgs` is the pure, unit-tested argument seam; `runBenchCommand` is the live boundary that resolves host access, resolves the scored suite against the host's self-review support, selects the task, and drives `runCells`. Invoked via `npm run bench:run` (see below).
|
||||
- `aggregate.ts` — the aggregator: the pure seam that renders the accumulated sample store into a readable comparison. `readAllSamples` drains a store into a flat record list; `aggregate` rolls the records up against the task definitions into a `Report` (headline, per-tier and per-token-component breakdowns, and the bonus table); `renderReport` renders that report as stable text. The headline metric, `costEquivalentTokens`, is computed here at render time by weighting the four retained token components by ADR 0014's pricing ratios (fresh input 1×, cache-write 1.25×, cache-read 0.1×, output 5×), so the stored records can be re-weighted without re-running. It reads whatever samples exist and annotates incomplete coverage — cells below the reporting floor (partial) and unsampled cells (missing) — rather than blocking on a complete matrix. A pure function of the records plus the definitions, unit-tested against synthetic append-only sample stores.
|
||||
|
||||
Later slices add the aggregator.
|
||||
The aggregator has no command wrapper yet; a `bench:report` CLI over it is a natural follow-up, in the same shape as `run.ts` is over `run-loop.ts`.
|
||||
|
||||
## Running a cell
|
||||
|
||||
|
||||
553
bench/aggregate.test.ts
Normal file
553
bench/aggregate.test.ts
Normal file
@@ -0,0 +1,553 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
aggregate,
|
||||
costEquivalentTokens,
|
||||
readAllSamples,
|
||||
renderReport,
|
||||
} from "./aggregate.js";
|
||||
import type { ResultRecord, TokenComponents } from "./result.js";
|
||||
import { createSampleStore } from "./store.js";
|
||||
|
||||
/** A minimal ResultRecord for the gitea-axi arm, overridable per sample. */
|
||||
function record(overrides: Partial<ResultRecord> = {}): ResultRecord {
|
||||
return {
|
||||
arm: "gitea-axi",
|
||||
taskId: "t1",
|
||||
tier: "read",
|
||||
trial: 1,
|
||||
timestamp: "2026-07-16T00:00:00Z",
|
||||
tokens: { freshInput: 0, cacheCreation: 0, cacheRead: 0, output: 0 },
|
||||
turns: 0,
|
||||
durationMs: 0,
|
||||
imputedCostUsd: 0,
|
||||
outcome: { pass: true },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("costEquivalentTokens", () => {
|
||||
// Behavior: cost-equivalent tokens weight a run's four retained token
|
||||
// components by Anthropic's published API pricing ratios — fresh input 1x,
|
||||
// cache-write 1.25x, cache-read 0.1x, output 5x (ADR 0014). The expected
|
||||
// total is derived BY HAND from those ratios, not recomputed the way the
|
||||
// code would, so it independently pins the metric.
|
||||
//
|
||||
// Distinct components are chosen so each of the four weighted products is a
|
||||
// different number and no two raw components share a value; a wrong weight on
|
||||
// any single component therefore cannot be masked by another:
|
||||
// freshInput 400 * 1 = 400
|
||||
// cacheCreation 800 * 1.25 = 1000
|
||||
// cacheRead 2000 * 0.1 = 200
|
||||
// output 600 * 5 = 3000
|
||||
// total = 4600
|
||||
it("weights the four retained token components by the documented pricing ratios", () => {
|
||||
const tokens: TokenComponents = {
|
||||
freshInput: 400,
|
||||
cacheCreation: 800,
|
||||
cacheRead: 2000,
|
||||
output: 600,
|
||||
};
|
||||
|
||||
expect(costEquivalentTokens(tokens)).toBe(4600);
|
||||
});
|
||||
});
|
||||
|
||||
describe("aggregate", () => {
|
||||
// Behavior: the headline view yields one row per arm whose per-run metrics —
|
||||
// cost-equivalent tokens (the headline), raw tokens, turns, duration, and
|
||||
// imputed cost — are the MEANS across that arm's samples, and whose success
|
||||
// rate is the fraction of its runs that passed. Here the gitea-axi arm has
|
||||
// two samples on task t1 (one pass, one fail). Every expected number below is
|
||||
// derived BY HAND from the two samples and the ADR 0014 pricing ratios
|
||||
// (fresh input 1x, cache-write 1.25x, cache-read 0.1x, output 5x), so it
|
||||
// pins the metric independently of how aggregate computes it:
|
||||
// cost-equiv A = 100*1 + 20*5 = 200; B = 200*1 + 40*5 = 400; mean = 300
|
||||
// raw tokens A = 120; B = 240; mean = 180
|
||||
// turns (5 + 7) / 2 = 6
|
||||
// durationMs (1000 + 3000) / 2 = 2000
|
||||
// successRate 1 pass of 2 = 0.5
|
||||
// imputedCost (0.02 + 0.06) / 2 = 0.04
|
||||
it("yields one headline row per arm with per-run means and the passing fraction as success rate", () => {
|
||||
const sampleA = record({
|
||||
trial: 1,
|
||||
tokens: { freshInput: 100, cacheCreation: 0, cacheRead: 0, output: 20 },
|
||||
turns: 5,
|
||||
durationMs: 1000,
|
||||
imputedCostUsd: 0.02,
|
||||
outcome: { pass: true },
|
||||
});
|
||||
const sampleB = record({
|
||||
trial: 2,
|
||||
tokens: { freshInput: 200, cacheCreation: 0, cacheRead: 0, output: 40 },
|
||||
turns: 7,
|
||||
durationMs: 3000,
|
||||
imputedCostUsd: 0.06,
|
||||
outcome: { pass: false, failure: "incorrect" },
|
||||
});
|
||||
|
||||
const report = aggregate({
|
||||
records: [sampleA, sampleB],
|
||||
suite: [{ id: "t1", tier: "read" }],
|
||||
bonus: [],
|
||||
});
|
||||
|
||||
const row = report.headline.find((r) => r.arm === "gitea-axi");
|
||||
if (!row) throw new Error("expected a gitea-axi headline row");
|
||||
|
||||
expect(row.samples).toBe(2);
|
||||
expect(row.costEquivalentTokens).toBeCloseTo(300);
|
||||
expect(row.rawTokens).toBeCloseTo(180);
|
||||
expect(row.turns).toBeCloseTo(6);
|
||||
expect(row.durationMs).toBeCloseTo(2000);
|
||||
expect(row.successRate).toBeCloseTo(0.5);
|
||||
expect(row.imputedCostUsd).toBeCloseTo(0.04);
|
||||
});
|
||||
|
||||
// Behavior: the per-tier breakdown is rendered from the same records — grouped
|
||||
// by tier, then by arm — reporting each tier-arm's cost-equivalent-token mean
|
||||
// and passing fraction. Three gitea-axi samples span two tiers: two on a read
|
||||
// task r1 (one pass, one fail) and one on a single-mutation task m1 (pass).
|
||||
// Every expected number is derived BY HAND from those samples and the ADR 0014
|
||||
// weights (fresh input 1x, output 5x; no cache here), independent of how
|
||||
// aggregate computes them:
|
||||
// read gitea-axi: cost A = 100 + 20*5 = 200; B = 200 + 40*5 = 400;
|
||||
// mean = 300; success 1 of 2 = 0.5; samples 2
|
||||
// single-mutation gitea-axi: cost = 50 + 10*5 = 100; success 1 of 1 = 1;
|
||||
// samples 1
|
||||
// The tier order is the fixed reporting order, an independent literal.
|
||||
it("breaks records down per tier and per arm with cost-equivalent-token means and success rates", () => {
|
||||
const r1a = record({
|
||||
taskId: "r1",
|
||||
tier: "read",
|
||||
trial: 1,
|
||||
tokens: { freshInput: 100, cacheCreation: 0, cacheRead: 0, output: 20 },
|
||||
outcome: { pass: true },
|
||||
});
|
||||
const r1b = record({
|
||||
taskId: "r1",
|
||||
tier: "read",
|
||||
trial: 2,
|
||||
tokens: { freshInput: 200, cacheCreation: 0, cacheRead: 0, output: 40 },
|
||||
outcome: { pass: false, failure: "incorrect" },
|
||||
});
|
||||
const m1c = record({
|
||||
taskId: "m1",
|
||||
tier: "single-mutation",
|
||||
trial: 1,
|
||||
tokens: { freshInput: 50, cacheCreation: 0, cacheRead: 0, output: 10 },
|
||||
outcome: { pass: true },
|
||||
});
|
||||
|
||||
const report = aggregate({
|
||||
records: [r1a, r1b, m1c],
|
||||
suite: [
|
||||
{ id: "r1", tier: "read" },
|
||||
{ id: "m1", tier: "single-mutation" },
|
||||
],
|
||||
bonus: [],
|
||||
});
|
||||
|
||||
// One breakdown per tier, in the fixed reporting order.
|
||||
expect(report.tiers.map((t) => t.tier)).toEqual([
|
||||
"read",
|
||||
"single-mutation",
|
||||
"find-then-act",
|
||||
"multi-step",
|
||||
]);
|
||||
|
||||
const readAxi = report.tiers
|
||||
.find((t) => t.tier === "read")
|
||||
?.arms.find((a) => a.arm === "gitea-axi");
|
||||
if (!readAxi) throw new Error("expected a read/gitea-axi tier-arm");
|
||||
expect(readAxi.samples).toBe(2);
|
||||
expect(readAxi.costEquivalentTokens).toBeCloseTo(300);
|
||||
expect(readAxi.successRate).toBeCloseTo(0.5);
|
||||
|
||||
const mutAxi = report.tiers
|
||||
.find((t) => t.tier === "single-mutation")
|
||||
?.arms.find((a) => a.arm === "gitea-axi");
|
||||
if (!mutAxi) throw new Error("expected a single-mutation/gitea-axi tier-arm");
|
||||
expect(mutAxi.samples).toBe(1);
|
||||
expect(mutAxi.costEquivalentTokens).toBeCloseTo(100);
|
||||
expect(mutAxi.successRate).toBeCloseTo(1);
|
||||
});
|
||||
|
||||
// Behavior: the per-token-component breakdown is rendered from the same
|
||||
// records — per arm, the per-run MEAN of each of the four raw token
|
||||
// components — so a reader can see what drives an arm's cost. The gitea-axi
|
||||
// arm has two samples; an arm with no samples reports null for every
|
||||
// component. Each mean is derived BY HAND from the two samples, independent
|
||||
// of how aggregate computes it:
|
||||
// freshInput (100 + 200) / 2 = 150
|
||||
// cacheCreation (40 + 60) / 2 = 50
|
||||
// cacheRead (1000 + 3000)/2 = 2000
|
||||
// output (20 + 40) / 2 = 30
|
||||
it("breaks records down per arm into the per-run mean of each token component", () => {
|
||||
const sampleA = record({
|
||||
trial: 1,
|
||||
tokens: { freshInput: 100, cacheCreation: 40, cacheRead: 1000, output: 20 },
|
||||
});
|
||||
const sampleB = record({
|
||||
trial: 2,
|
||||
tokens: { freshInput: 200, cacheCreation: 60, cacheRead: 3000, output: 40 },
|
||||
});
|
||||
|
||||
const report = aggregate({
|
||||
records: [sampleA, sampleB],
|
||||
suite: [{ id: "t1", tier: "read" }],
|
||||
bonus: [],
|
||||
});
|
||||
|
||||
const axi = report.components.find((c) => c.arm === "gitea-axi");
|
||||
if (!axi) throw new Error("expected a gitea-axi component breakdown");
|
||||
expect(axi.freshInput).toBeCloseTo(150);
|
||||
expect(axi.cacheCreation).toBeCloseTo(50);
|
||||
expect(axi.cacheRead).toBeCloseTo(2000);
|
||||
expect(axi.output).toBeCloseTo(30);
|
||||
|
||||
// An arm with no samples reports null for every component.
|
||||
const tea = report.components.find((c) => c.arm === "tea");
|
||||
if (!tea) throw new Error("expected a tea component breakdown");
|
||||
expect(tea.freshInput).toBeNull();
|
||||
expect(tea.cacheCreation).toBeNull();
|
||||
expect(tea.cacheRead).toBeNull();
|
||||
expect(tea.output).toBeNull();
|
||||
});
|
||||
|
||||
// Behavior: the separate bonus table is rendered from the same records. It
|
||||
// emits one row per supplied bonus definition, in the given order, carrying
|
||||
// that operation's capability metadata (operation text, direction, note, and
|
||||
// gitea-axi's own applicability), plus — for any arm that actually has
|
||||
// samples for that bonus cell — the per-arm run metrics. Bonus cells are
|
||||
// usually unrun, so a cell's arms list is empty unless records exist for it.
|
||||
// Here bonus-x has one tea sample and bonus-y has none. The one derived
|
||||
// number is by hand from that sample and the ADR 0014 weights (fresh input
|
||||
// 1x, output 5x): cost = 100 + 20*5 = 200.
|
||||
it("emits one bonus row per definition with its capability metadata and per-arm samples only where run", () => {
|
||||
const teaSample = record({
|
||||
arm: "tea",
|
||||
taskId: "bonus-x",
|
||||
tier: "read",
|
||||
trial: 1,
|
||||
tokens: { freshInput: 100, cacheCreation: 0, cacheRead: 0, output: 20 },
|
||||
outcome: { pass: true },
|
||||
});
|
||||
|
||||
const report = aggregate({
|
||||
records: [teaSample],
|
||||
suite: [],
|
||||
bonus: [
|
||||
{
|
||||
id: "bonus-x",
|
||||
operation: "Do the X thing",
|
||||
direction: "gitea-axi-advantage",
|
||||
note: "only gitea-axi does X ergonomically",
|
||||
giteaAxi: "applicable",
|
||||
},
|
||||
{
|
||||
id: "bonus-y",
|
||||
operation: "Do the Y thing",
|
||||
direction: "gitea-axi-not-applicable",
|
||||
note: "Y is outside gitea-axi's surface",
|
||||
giteaAxi: "not-applicable",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// One row per definition, in the given order.
|
||||
expect(report.bonus.map((b) => b.id)).toEqual(["bonus-x", "bonus-y"]);
|
||||
|
||||
// bonus-x carries its capability metadata and the tea cell's metrics.
|
||||
const bx = report.bonus.find((b) => b.id === "bonus-x");
|
||||
if (!bx) throw new Error("expected a bonus-x row");
|
||||
expect(bx.operation).toBe("Do the X thing");
|
||||
expect(bx.direction).toBe("gitea-axi-advantage");
|
||||
expect(bx.note).toBe("only gitea-axi does X ergonomically");
|
||||
expect(bx.giteaAxi).toBe("applicable");
|
||||
|
||||
const bxTea = bx.arms.find((a) => a.arm === "tea");
|
||||
if (!bxTea) throw new Error("expected a tea entry on bonus-x");
|
||||
expect(bx.arms).toHaveLength(1);
|
||||
expect(bxTea.samples).toBe(1);
|
||||
expect(bxTea.costEquivalentTokens).toBeCloseTo(200);
|
||||
expect(bxTea.successRate).toBeCloseTo(1);
|
||||
|
||||
// bonus-y was never run, so it carries metadata but no arm metrics.
|
||||
const by = report.bonus.find((b) => b.id === "bonus-y");
|
||||
if (!by) throw new Error("expected a bonus-y row");
|
||||
expect(by.giteaAxi).toBe("not-applicable");
|
||||
expect(by.direction).toBe("gitea-axi-not-applicable");
|
||||
expect(by.arms).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderReport", () => {
|
||||
// Behavior: the rendered headline table shows one row per arm, presents
|
||||
// cost-equivalent tokens as the headline metric column and imputed cost as a
|
||||
// de-emphasized secondary column, and marks an arm with no samples with the
|
||||
// "—" placeholder. Two gitea-axi samples on t1 give a cost-equivalent mean of
|
||||
// (200 + 400)/2 = 300 and an imputed-cost mean of (0.02 + 0.06)/2 = 0.04,
|
||||
// both derived BY HAND, independent of how the report is rendered. Assertions
|
||||
// target the gitea-axi row's own line and the presence of labels/values —
|
||||
// never fixed column widths — so they survive a layout refactor.
|
||||
it("renders one row per arm with cost-equivalent tokens as the headline and imputed cost as a secondary column", () => {
|
||||
const sampleA = record({
|
||||
trial: 1,
|
||||
tokens: { freshInput: 100, cacheCreation: 0, cacheRead: 0, output: 20 },
|
||||
turns: 5,
|
||||
durationMs: 1000,
|
||||
imputedCostUsd: 0.02,
|
||||
outcome: { pass: true },
|
||||
});
|
||||
const sampleB = record({
|
||||
trial: 2,
|
||||
tokens: { freshInput: 200, cacheCreation: 0, cacheRead: 0, output: 40 },
|
||||
turns: 7,
|
||||
durationMs: 3000,
|
||||
imputedCostUsd: 0.06,
|
||||
outcome: { pass: false, failure: "incorrect" },
|
||||
});
|
||||
|
||||
const output = renderReport(
|
||||
aggregate({
|
||||
records: [sampleA, sampleB],
|
||||
suite: [{ id: "t1", tier: "read" }],
|
||||
bonus: [],
|
||||
}),
|
||||
);
|
||||
|
||||
// Every arm gets a row.
|
||||
expect(output).toContain("gitea-axi");
|
||||
expect(output).toContain("tea");
|
||||
expect(output).toContain("gitea-mcp");
|
||||
expect(output).toContain("raw-api");
|
||||
|
||||
// The headline metric and the secondary column are labelled.
|
||||
expect(output.toLowerCase()).toContain("cost-equivalent");
|
||||
expect(output.toLowerCase()).toContain("imputed");
|
||||
|
||||
// The gitea-axi row carries its cost-equivalent mean (300) and imputed
|
||||
// mean (0.04) on its own line.
|
||||
const axiLine = output.split("\n").find((l) => l.includes("gitea-axi"));
|
||||
if (!axiLine) throw new Error("expected a gitea-axi row line");
|
||||
expect(axiLine).toContain("300");
|
||||
expect(axiLine).toContain("0.04");
|
||||
|
||||
// An arm with no samples shows the em-dash placeholder on its row. The
|
||||
// "tea" arm token is matched at a word boundary so the substring inside
|
||||
// "gitea-axi"/"gitea-mcp" cannot be mistaken for the tea row.
|
||||
const teaLine = output
|
||||
.split("\n")
|
||||
.find((l) => /(^|[^a-z-])tea([^a-z-]|$)/.test(l));
|
||||
if (!teaLine) throw new Error("expected a tea row line");
|
||||
expect(teaLine).toContain("—");
|
||||
});
|
||||
|
||||
// Behavior: a partial matrix renders without error and incomplete coverage is
|
||||
// annotated rather than hidden or treated as complete. With the default
|
||||
// reporting floor of 3, a cell is covered at >=3 samples, partial at 1-2, and
|
||||
// missing at 0. gitea-axi has one covered / one partial / one missing cell,
|
||||
// so it is incomplete; tea has all three cells covered, so it is complete.
|
||||
// The coverage counts (1 partial, 1 missing for gitea-axi) are derived BY
|
||||
// HAND from the sample layout, independent of how the report is rendered.
|
||||
// Lines are located by content, not index, so the assertions survive a
|
||||
// layout refactor.
|
||||
it("renders a partial matrix without throwing and annotates incomplete coverage rather than hiding it", () => {
|
||||
const read = (taskId: string, trial: number) =>
|
||||
record({ taskId, tier: "read", trial, outcome: { pass: true } });
|
||||
|
||||
const records: ResultRecord[] = [
|
||||
// gitea-axi: t1 covered (3), t2 partial (2), t3 missing (0).
|
||||
read("t1", 1),
|
||||
read("t1", 2),
|
||||
read("t1", 3),
|
||||
read("t2", 1),
|
||||
read("t2", 2),
|
||||
// tea: all three tasks covered (3 each).
|
||||
{ ...read("t1", 1), arm: "tea" },
|
||||
{ ...read("t1", 2), arm: "tea" },
|
||||
{ ...read("t1", 3), arm: "tea" },
|
||||
{ ...read("t2", 1), arm: "tea" },
|
||||
{ ...read("t2", 2), arm: "tea" },
|
||||
{ ...read("t2", 3), arm: "tea" },
|
||||
{ ...read("t3", 1), arm: "tea" },
|
||||
{ ...read("t3", 2), arm: "tea" },
|
||||
{ ...read("t3", 3), arm: "tea" },
|
||||
];
|
||||
|
||||
const input = {
|
||||
records,
|
||||
suite: [
|
||||
{ id: "t1", tier: "read" as const },
|
||||
{ id: "t2", tier: "read" as const },
|
||||
{ id: "t3", tier: "read" as const },
|
||||
],
|
||||
bonus: [],
|
||||
};
|
||||
|
||||
// A partial matrix renders without error.
|
||||
expect(() => renderReport(aggregate(input))).not.toThrow();
|
||||
|
||||
const output = renderReport(aggregate(input));
|
||||
|
||||
// The coverage annotation names the reporting floor it is measured against.
|
||||
expect(output).toContain("reporting floor of 3");
|
||||
|
||||
// gitea-axi is incomplete: 1 partial, 1 missing, flagged incomplete.
|
||||
const axiLine = output
|
||||
.split("\n")
|
||||
.find((l) => /\bgitea-axi\b/.test(l) && l.includes("partial"));
|
||||
if (!axiLine) throw new Error("expected a gitea-axi coverage annotation");
|
||||
expect(axiLine).toContain("1 partial");
|
||||
expect(axiLine).toContain("1 missing");
|
||||
expect(axiLine).toContain("incomplete");
|
||||
|
||||
// tea is complete and is not labelled incomplete.
|
||||
const teaLine = output
|
||||
.split("\n")
|
||||
.find((l) => /(^|[^a-z-])tea([^a-z-]|$)/.test(l) && l.includes("complete"));
|
||||
if (!teaLine) throw new Error("expected a tea coverage annotation");
|
||||
expect(teaLine).toContain("complete");
|
||||
expect(teaLine).not.toContain("incomplete");
|
||||
});
|
||||
|
||||
// Behavior: the per-tier breakdown, the per-token-component breakdown, and the
|
||||
// separate bonus table are all rendered from the same records — each as its
|
||||
// own section surfacing its content. A single gitea-axi sample on a read task
|
||||
// fixes the component means (one sample, so each mean equals its raw value):
|
||||
// fresh input 300, cache write 80, cache read 1000, output 60 — derived BY
|
||||
// HAND, independent of how the report renders. Assertions are presence-based
|
||||
// (section headers and content strings), not column layout, so they survive a
|
||||
// rendering refactor.
|
||||
it("renders the per-tier, per-token-component, and bonus sections from the same records", () => {
|
||||
const output = renderReport(
|
||||
aggregate({
|
||||
records: [
|
||||
record({
|
||||
taskId: "r1",
|
||||
tier: "read",
|
||||
trial: 1,
|
||||
tokens: {
|
||||
freshInput: 300,
|
||||
cacheCreation: 80,
|
||||
cacheRead: 1000,
|
||||
output: 60,
|
||||
},
|
||||
outcome: { pass: true },
|
||||
}),
|
||||
],
|
||||
suite: [
|
||||
{ id: "r1", tier: "read" },
|
||||
{ id: "m1", tier: "single-mutation" },
|
||||
],
|
||||
bonus: [
|
||||
{
|
||||
id: "bonus-x",
|
||||
operation: "Do the X thing",
|
||||
direction: "gitea-axi-advantage",
|
||||
note: "only gitea-axi does X",
|
||||
giteaAxi: "applicable",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const lower = output.toLowerCase();
|
||||
|
||||
// Per-tier section: a header naming tiers, and every tier surfaced.
|
||||
expect(lower).toContain("tier");
|
||||
expect(output).toContain("read");
|
||||
expect(output).toContain("single-mutation");
|
||||
expect(output).toContain("find-then-act");
|
||||
expect(output).toContain("multi-step");
|
||||
|
||||
// Per-token-component section: a header, the four component labels, and
|
||||
// gitea-axi's component means.
|
||||
expect(lower).toContain("component");
|
||||
expect(lower).toContain("fresh input");
|
||||
expect(lower).toContain("cache write");
|
||||
expect(lower).toContain("cache read");
|
||||
expect(lower).toContain("output");
|
||||
expect(output).toContain("300");
|
||||
expect(output).toContain("80");
|
||||
expect(output).toContain("1000");
|
||||
expect(output).toContain("60");
|
||||
|
||||
// Bonus section: a header, the operation text, and gitea-axi's applicability.
|
||||
expect(lower).toContain("bonus");
|
||||
expect(output).toContain("Do the X thing");
|
||||
expect(output).toContain("applicable");
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderReport over an append-only sample store", () => {
|
||||
let rootA: string;
|
||||
let rootB: string;
|
||||
|
||||
beforeEach(() => {
|
||||
rootA = mkdtempSync(join(tmpdir(), "bench-aggregate-"));
|
||||
rootB = mkdtempSync(join(tmpdir(), "bench-aggregate-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(rootA, { recursive: true, force: true });
|
||||
rmSync(rootB, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// Behavior: rendering is stable — the rendered report is a function of the
|
||||
// accumulated set of samples only, so the order in which samples were
|
||||
// appended to the store must not change the output. This is a stability
|
||||
// property a correct implementation already satisfies, exercised end to end
|
||||
// through the real append-only store. The same records are appended to two
|
||||
// independent stores in genuinely different orders (one reversed and with
|
||||
// cells interleaved differently); the two rendered reports must be identical.
|
||||
it("renders identically regardless of the order samples were appended", () => {
|
||||
// A set spanning two arms and two tasks; the gitea-axi/t1 cell reaches the
|
||||
// reporting floor of three so the headline and coverage sections are
|
||||
// non-trivial rather than all placeholders.
|
||||
const records: ResultRecord[] = [
|
||||
record({ arm: "gitea-axi", taskId: "t1", tier: "read", trial: 1 }),
|
||||
record({ arm: "gitea-axi", taskId: "t1", tier: "read", trial: 2 }),
|
||||
record({ arm: "gitea-axi", taskId: "t1", tier: "read", trial: 3 }),
|
||||
record({ arm: "gitea-axi", taskId: "t2", tier: "single-mutation", trial: 1 }),
|
||||
record({ arm: "tea", taskId: "t1", tier: "read", trial: 1 }),
|
||||
record({ arm: "tea", taskId: "t2", tier: "single-mutation", trial: 1 }),
|
||||
record({ arm: "tea", taskId: "t2", tier: "single-mutation", trial: 2 }),
|
||||
];
|
||||
|
||||
// Two genuinely different append orders over the same set.
|
||||
const orderA = records;
|
||||
const orderB = [...records].reverse();
|
||||
// The two orders truly differ, so the stability claim is not vacuous.
|
||||
expect(orderB).not.toEqual(orderA);
|
||||
|
||||
const storeA = createSampleStore(rootA);
|
||||
for (const r of orderA) storeA.append(r);
|
||||
|
||||
const storeB = createSampleStore(rootB);
|
||||
for (const r of orderB) storeB.append(r);
|
||||
|
||||
const suite = [
|
||||
{ id: "t1", tier: "read" as const },
|
||||
{ id: "t2", tier: "single-mutation" as const },
|
||||
];
|
||||
|
||||
const a = renderReport(
|
||||
aggregate({ records: readAllSamples(storeA), suite, bonus: [] }),
|
||||
);
|
||||
const b = renderReport(
|
||||
aggregate({ records: readAllSamples(storeB), suite, bonus: [] }),
|
||||
);
|
||||
|
||||
// Append order does not affect the rendered report.
|
||||
expect(a).toBe(b);
|
||||
|
||||
// Non-vacuity: the report actually rendered real headline content.
|
||||
expect(a.length).toBeGreaterThan(0);
|
||||
expect(a).toContain("gitea-axi");
|
||||
expect(a.toLowerCase()).toContain("cost-equivalent");
|
||||
});
|
||||
});
|
||||
457
bench/aggregate.ts
Normal file
457
bench/aggregate.ts
Normal file
@@ -0,0 +1,457 @@
|
||||
// The aggregator: the pure seam that renders the accumulated sample store into a
|
||||
// readable comparison. It reads whatever samples exist and annotates incomplete
|
||||
// coverage rather than blocking on a complete matrix, so a half-run benchmark
|
||||
// still produces a non-misleading table.
|
||||
//
|
||||
// The headline metric — cost-equivalent tokens — is computed here at render time
|
||||
// by weighting each run's four retained token components by Anthropic's published
|
||||
// API pricing ratios (see ADR 0014). Nothing is pre-summed in the stored records,
|
||||
// so the data can be re-weighted without re-running if the subscription's weekly
|
||||
// accounting is ever documented.
|
||||
//
|
||||
// This module is a pure function of the records plus the task definitions: no I/O
|
||||
// beyond `readAllSamples`, which drains a sample store into a flat record list.
|
||||
// Everything else — aggregate and renderReport — is deterministic and unit-tested
|
||||
// against synthetic sample stores.
|
||||
|
||||
import type { Arm, ResultRecord, Tier, TokenComponents } from "./result.js";
|
||||
import { REPORTING_FLOOR } from "./run-loop.js";
|
||||
import type { SampleStore } from "./store.js";
|
||||
import type { BenchTask } from "./task.js";
|
||||
import type { Applicability, BonusDirection, BonusTask } from "./task-suite.js";
|
||||
|
||||
/**
|
||||
* The cost-equivalent-token weights, from ADR 0014: fresh input 1×, cache-write
|
||||
* 1.25× (the 5-minute-TTL cache-write multiplier — the records retain a single
|
||||
* un-TTL'd cache-creation component, so the default write price applies),
|
||||
* cache-read 0.1×, output 5×. These are Anthropic's published API pricing ratios;
|
||||
* changing the subscription's real accounting means changing only these numbers,
|
||||
* since the stored records keep the four components un-weighted.
|
||||
*/
|
||||
export const COST_EQUIVALENT_WEIGHTS: Readonly<Record<keyof TokenComponents, number>> = {
|
||||
freshInput: 1,
|
||||
cacheCreation: 1.25,
|
||||
cacheRead: 0.1,
|
||||
output: 5,
|
||||
};
|
||||
|
||||
/** The fixed arm display order; every headline and breakdown lists arms in it. */
|
||||
export const ARM_ORDER: readonly Arm[] = ["gitea-axi", "tea", "gitea-mcp", "raw-api"];
|
||||
|
||||
/** The fixed tier display order for the per-tier breakdown. */
|
||||
export const TIER_ORDER: readonly Tier[] = [
|
||||
"read",
|
||||
"single-mutation",
|
||||
"find-then-act",
|
||||
"multi-step",
|
||||
];
|
||||
|
||||
/** Weight one run's four token components into a single cost-equivalent token count. */
|
||||
export function costEquivalentTokens(tokens: TokenComponents): number {
|
||||
return (
|
||||
tokens.freshInput * COST_EQUIVALENT_WEIGHTS.freshInput +
|
||||
tokens.cacheCreation * COST_EQUIVALENT_WEIGHTS.cacheCreation +
|
||||
tokens.cacheRead * COST_EQUIVALENT_WEIGHTS.cacheRead +
|
||||
tokens.output * COST_EQUIVALENT_WEIGHTS.output
|
||||
);
|
||||
}
|
||||
|
||||
/** Sum one run's four token components at 1× — the raw, assumption-free total. */
|
||||
export function rawTokens(tokens: TokenComponents): number {
|
||||
return tokens.freshInput + tokens.cacheCreation + tokens.cacheRead + tokens.output;
|
||||
}
|
||||
|
||||
/**
|
||||
* How much of an arm's slice of the matrix has been run. A cell is one task; it is
|
||||
* `covered` once it holds at least the reporting floor of samples, `partial` while
|
||||
* it holds fewer, and `missing` while it holds none. The three always sum to
|
||||
* `tasksTotal`, so partial and missing coverage is annotated rather than hidden.
|
||||
*/
|
||||
export interface Coverage {
|
||||
tasksTotal: number;
|
||||
covered: number;
|
||||
partial: number;
|
||||
missing: number;
|
||||
}
|
||||
|
||||
/** One headline row: an arm's per-run means plus its success rate and coverage. */
|
||||
export interface ArmHeadline {
|
||||
arm: Arm;
|
||||
/** Samples the arm holds across all its cells. */
|
||||
samples: number;
|
||||
/** Mean cost-equivalent tokens per run; `null` when the arm has no samples. */
|
||||
costEquivalentTokens: number | null;
|
||||
/** Mean raw token sum per run; `null` when the arm has no samples. */
|
||||
rawTokens: number | null;
|
||||
/** Mean turns per run; `null` when the arm has no samples. */
|
||||
turns: number | null;
|
||||
/** Mean wall-clock duration (ms) per run; `null` when the arm has no samples. */
|
||||
durationMs: number | null;
|
||||
/** Fraction of runs that passed (0..1); `null` when the arm has no samples. */
|
||||
successRate: number | null;
|
||||
/** Mean imputed cost (USD) per run; `null` when the arm has no samples. */
|
||||
imputedCostUsd: number | null;
|
||||
coverage: Coverage;
|
||||
}
|
||||
|
||||
/** One arm's slice of a per-tier breakdown. */
|
||||
export interface TierArm {
|
||||
arm: Arm;
|
||||
samples: number;
|
||||
costEquivalentTokens: number | null;
|
||||
successRate: number | null;
|
||||
coverage: Coverage;
|
||||
}
|
||||
|
||||
/** The per-tier breakdown for one tier, one row per arm. */
|
||||
export interface TierBreakdown {
|
||||
tier: Tier;
|
||||
arms: TierArm[];
|
||||
}
|
||||
|
||||
/** One arm's mean per-run token component breakdown; `null` fields when no samples. */
|
||||
export interface ComponentBreakdown {
|
||||
arm: Arm;
|
||||
freshInput: number | null;
|
||||
cacheCreation: number | null;
|
||||
cacheRead: number | null;
|
||||
output: number | null;
|
||||
}
|
||||
|
||||
/** One arm's run metrics for a bonus operation, when any samples exist for it. */
|
||||
export interface BonusArmSamples {
|
||||
arm: Arm;
|
||||
samples: number;
|
||||
costEquivalentTokens: number | null;
|
||||
successRate: number | null;
|
||||
}
|
||||
|
||||
/** One row of the bonus table: a capability-asymmetric operation and its status. */
|
||||
export interface BonusRow {
|
||||
id: string;
|
||||
operation: string;
|
||||
direction: BonusDirection;
|
||||
note: string;
|
||||
/** gitea-axi's own applicability for the operation, from the definition. */
|
||||
giteaAxi: Applicability;
|
||||
/** Per-arm run metrics from the records; usually empty (bonus cells are rarely run). */
|
||||
arms: BonusArmSamples[];
|
||||
}
|
||||
|
||||
/** The fully-aggregated report, ready to render. */
|
||||
export interface Report {
|
||||
/** The floor a cell's sample count must reach to count as covered. */
|
||||
reportingFloor: number;
|
||||
headline: ArmHeadline[];
|
||||
tiers: TierBreakdown[];
|
||||
components: ComponentBreakdown[];
|
||||
bonus: BonusRow[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The task facts the aggregator needs to score coverage: an id to key each cell
|
||||
* and a tier to group it under. The real scored suite (`BenchTask[]`) satisfies
|
||||
* this, but so does a bare `{ id, tier }` list, since the aggregator never scores.
|
||||
*/
|
||||
export type TaskCoverage = Pick<BenchTask, "id" | "tier">;
|
||||
|
||||
/** Everything the aggregator needs: the samples and the definitions to score coverage against. */
|
||||
export interface AggregateInput {
|
||||
records: readonly ResultRecord[];
|
||||
suite: readonly TaskCoverage[];
|
||||
bonus: readonly BonusTask[];
|
||||
/** Samples a cell needs before it counts as covered; defaults to {@link REPORTING_FLOOR}. */
|
||||
reportingFloor?: number;
|
||||
}
|
||||
|
||||
/** Drain every sample the store holds into one flat, append-order record list. */
|
||||
export function readAllSamples(store: SampleStore): ResultRecord[] {
|
||||
return store.cells().flatMap((cell) => store.read(cell));
|
||||
}
|
||||
|
||||
/** The arithmetic mean of a list, or `null` for an empty list (no data to average). */
|
||||
function mean(values: number[]): number | null {
|
||||
if (values.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
/** The fraction of runs that passed, or `null` when there are no runs. */
|
||||
function passRate(records: readonly ResultRecord[]): number | null {
|
||||
if (records.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return records.filter((record) => record.outcome.pass).length / records.length;
|
||||
}
|
||||
|
||||
/** Mean cost-equivalent tokens per run over a record set, or `null` when empty. */
|
||||
function meanCostEquivalent(records: readonly ResultRecord[]): number | null {
|
||||
return mean(records.map((record) => costEquivalentTokens(record.tokens)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Count how a task set's cells are covered by one arm's records: a cell is
|
||||
* `covered` once it holds at least the floor of samples, `partial` below it,
|
||||
* `missing` at zero. The three always sum to the task count, so partial and
|
||||
* missing coverage is annotated rather than hidden.
|
||||
*/
|
||||
function coverageOf(
|
||||
tasks: readonly TaskCoverage[],
|
||||
armRecords: readonly ResultRecord[],
|
||||
floor: number,
|
||||
): Coverage {
|
||||
let covered = 0;
|
||||
let partial = 0;
|
||||
let missing = 0;
|
||||
for (const task of tasks) {
|
||||
const count = armRecords.filter((record) => record.taskId === task.id).length;
|
||||
if (count >= floor) {
|
||||
covered += 1;
|
||||
} else if (count > 0) {
|
||||
partial += 1;
|
||||
} else {
|
||||
missing += 1;
|
||||
}
|
||||
}
|
||||
return { tasksTotal: tasks.length, covered, partial, missing };
|
||||
}
|
||||
|
||||
/** Aggregate the records against the task definitions into a renderable report. */
|
||||
export function aggregate(input: AggregateInput): Report {
|
||||
const floor = input.reportingFloor ?? REPORTING_FLOOR;
|
||||
|
||||
const headline = ARM_ORDER.map((arm): ArmHeadline => {
|
||||
const armRecords = input.records.filter((record) => record.arm === arm);
|
||||
return {
|
||||
arm,
|
||||
samples: armRecords.length,
|
||||
costEquivalentTokens: meanCostEquivalent(armRecords),
|
||||
rawTokens: mean(armRecords.map((record) => rawTokens(record.tokens))),
|
||||
turns: mean(armRecords.map((record) => record.turns)),
|
||||
durationMs: mean(armRecords.map((record) => record.durationMs)),
|
||||
successRate: passRate(armRecords),
|
||||
imputedCostUsd: mean(armRecords.map((record) => record.imputedCostUsd)),
|
||||
coverage: coverageOf(input.suite, armRecords, floor),
|
||||
};
|
||||
});
|
||||
|
||||
const tiers = TIER_ORDER.map((tier): TierBreakdown => {
|
||||
const tierTasks = input.suite.filter((task) => task.tier === tier);
|
||||
const tierRecords = input.records.filter((record) => record.tier === tier);
|
||||
const arms = ARM_ORDER.map((arm): TierArm => {
|
||||
const armRecords = tierRecords.filter((record) => record.arm === arm);
|
||||
return {
|
||||
arm,
|
||||
samples: armRecords.length,
|
||||
costEquivalentTokens: meanCostEquivalent(armRecords),
|
||||
successRate: passRate(armRecords),
|
||||
coverage: coverageOf(tierTasks, armRecords, floor),
|
||||
};
|
||||
});
|
||||
return { tier, arms };
|
||||
});
|
||||
|
||||
const components = ARM_ORDER.map((arm): ComponentBreakdown => {
|
||||
const armRecords = input.records.filter((record) => record.arm === arm);
|
||||
return {
|
||||
arm,
|
||||
freshInput: mean(armRecords.map((record) => record.tokens.freshInput)),
|
||||
cacheCreation: mean(armRecords.map((record) => record.tokens.cacheCreation)),
|
||||
cacheRead: mean(armRecords.map((record) => record.tokens.cacheRead)),
|
||||
output: mean(armRecords.map((record) => record.tokens.output)),
|
||||
};
|
||||
});
|
||||
|
||||
const bonus = input.bonus.map((definition): BonusRow => {
|
||||
const cellRecords = input.records.filter((record) => record.taskId === definition.id);
|
||||
const arms = ARM_ORDER.flatMap((arm): BonusArmSamples[] => {
|
||||
const armRecords = cellRecords.filter((record) => record.arm === arm);
|
||||
if (armRecords.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
arm,
|
||||
samples: armRecords.length,
|
||||
costEquivalentTokens: meanCostEquivalent(armRecords),
|
||||
successRate: passRate(armRecords),
|
||||
},
|
||||
];
|
||||
});
|
||||
return {
|
||||
id: definition.id,
|
||||
operation: definition.operation,
|
||||
direction: definition.direction,
|
||||
note: definition.note,
|
||||
giteaAxi: definition.giteaAxi,
|
||||
arms,
|
||||
};
|
||||
});
|
||||
|
||||
return { reportingFloor: floor, headline, tiers, components, bonus };
|
||||
}
|
||||
|
||||
/** Per-column horizontal alignment for the text tables. */
|
||||
type Align = "left" | "right";
|
||||
|
||||
/** Render a value with `format`, or the em-dash placeholder when it is absent. */
|
||||
function cell(value: number | null, format: (value: number) => string): string {
|
||||
return value === null ? "—" : format(value);
|
||||
}
|
||||
|
||||
/** Round to a whole number — the scale token counts are reported at. */
|
||||
const asInteger = (value: number): string => String(Math.round(value));
|
||||
/** One decimal place — for the soft turn and (via seconds) duration metrics. */
|
||||
const asDecimal = (value: number): string => value.toFixed(1);
|
||||
/** Milliseconds as seconds, since durations are seconds-scale and network-soft. */
|
||||
const asSeconds = (value: number): string => `${(value / 1000).toFixed(1)}s`;
|
||||
/** A 0..1 rate as a whole-percent success figure. */
|
||||
const asPercent = (value: number): string => `${Math.round(value * 100)}%`;
|
||||
/** Imputed dollars, kept to cents and marked approximate and secondary. */
|
||||
const asImputedDollars = (value: number): string => `~$${value.toFixed(2)}`;
|
||||
/** A coverage figure as covered-of-total cells. */
|
||||
const asCoverage = (coverage: Coverage): string => `${coverage.covered}/${coverage.tasksTotal}`;
|
||||
|
||||
/**
|
||||
* Render a table as aligned, space-separated columns. Column widths fit their
|
||||
* widest cell, so rendering is stable for a given set of rows regardless of the
|
||||
* order they were accumulated in.
|
||||
*/
|
||||
function renderTable(headers: string[], aligns: Align[], rows: string[][]): string[] {
|
||||
const widths = headers.map((header, column) =>
|
||||
Math.max(header.length, ...rows.map((row) => (row[column] ?? "").length)),
|
||||
);
|
||||
const pad = (text: string, column: number): string =>
|
||||
aligns[column] === "right" ? text.padStart(widths[column]!) : text.padEnd(widths[column]!);
|
||||
const line = (values: string[]): string =>
|
||||
values.map((value, column) => pad(value, column)).join(" ").trimEnd();
|
||||
return [line(headers), ...rows.map(line)];
|
||||
}
|
||||
|
||||
/**
|
||||
* The headline table: one row per arm, cost-equivalent tokens as the headline
|
||||
* metric, then the raw token sum, turns, duration, success rate, and a coverage
|
||||
* figure, with imputed cost as a de-emphasized (parenthesized, approximate)
|
||||
* trailing column. Metric cells for an arm with no samples read as an em dash.
|
||||
*/
|
||||
function renderHeadline(report: Report): string[] {
|
||||
const headers = ["arm", "cost-eq", "raw", "turns", "duration", "success", "coverage", "(imputed $)"];
|
||||
const aligns: Align[] = ["left", "right", "right", "right", "right", "right", "right", "right"];
|
||||
const rows = report.headline.map((row) => [
|
||||
row.arm,
|
||||
cell(row.costEquivalentTokens, asInteger),
|
||||
cell(row.rawTokens, asInteger),
|
||||
cell(row.turns, asDecimal),
|
||||
cell(row.durationMs, asSeconds),
|
||||
cell(row.successRate, asPercent),
|
||||
asCoverage(row.coverage),
|
||||
cell(row.imputedCostUsd, asImputedDollars),
|
||||
]);
|
||||
return [
|
||||
"Headline — cost-equivalent tokens (the headline metric), one row per arm:",
|
||||
...renderTable(headers, aligns, rows),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The coverage annotation: one line per arm marking its coverage complete or
|
||||
* incomplete, with the partial (below-floor) and missing (unsampled) cell counts
|
||||
* spelled out. This is what keeps a half-run matrix from reading as a full one —
|
||||
* an arm is never silently presented as complete when cells are still missing.
|
||||
*/
|
||||
function renderCoverage(report: Report): string[] {
|
||||
const tasksTotal = report.headline[0]?.coverage.tasksTotal ?? 0;
|
||||
const header = `Coverage — cells at or above the reporting floor of ${report.reportingFloor} samples, out of ${tasksTotal} tasks per arm:`;
|
||||
const lines = report.headline.map((row) => {
|
||||
const { covered, tasksTotal: total, partial, missing } = row.coverage;
|
||||
if (partial === 0 && missing === 0) {
|
||||
return ` ${row.arm}: ${covered}/${total} covered — complete`;
|
||||
}
|
||||
return ` ${row.arm}: ${covered}/${total} covered, ${partial} partial, ${missing} missing — incomplete`;
|
||||
});
|
||||
return [header, ...lines];
|
||||
}
|
||||
|
||||
/**
|
||||
* The per-tier breakdown: for each tier, a small arm table of cost-equivalent
|
||||
* tokens (mean), success rate, and that tier's coverage. Shows where an arm wins
|
||||
* or loses across the tiers, derived from the same records as the headline.
|
||||
*/
|
||||
function renderTiers(report: Report): string[] {
|
||||
const headers = ["arm", "cost-eq", "success", "coverage"];
|
||||
const aligns: Align[] = ["left", "right", "right", "right"];
|
||||
const lines = ["Per-tier breakdown — cost-equivalent tokens (mean) and success rate by tier:"];
|
||||
for (const tier of report.tiers) {
|
||||
const rows = tier.arms.map((arm) => [
|
||||
arm.arm,
|
||||
cell(arm.costEquivalentTokens, asInteger),
|
||||
cell(arm.successRate, asPercent),
|
||||
asCoverage(arm.coverage),
|
||||
]);
|
||||
lines.push(` ${tier.tier}`);
|
||||
for (const line of renderTable(headers, aligns, rows)) {
|
||||
lines.push(` ${line}`);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/**
|
||||
* The per-token-component breakdown: each arm's mean tokens per run split into
|
||||
* the four retained components, so a reader can see what drives an arm's cost
|
||||
* (typically cache-read volume). The component labels name the pricing tiers the
|
||||
* cost-equivalent weights apply to.
|
||||
*/
|
||||
function renderComponents(report: Report): string[] {
|
||||
const headers = ["arm", "fresh input", "cache write", "cache read", "output"];
|
||||
const aligns: Align[] = ["left", "right", "right", "right", "right"];
|
||||
const rows = report.components.map((row) => [
|
||||
row.arm,
|
||||
cell(row.freshInput, asInteger),
|
||||
cell(row.cacheCreation, asInteger),
|
||||
cell(row.cacheRead, asInteger),
|
||||
cell(row.output, asInteger),
|
||||
]);
|
||||
return [
|
||||
"Per-token-component breakdown — mean tokens per run by component:",
|
||||
...renderTable(headers, aligns, rows),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The separate bonus table: the capability-asymmetric operations kept out of the
|
||||
* scored comparison, each with its direction, gitea-axi's own applicability, and a
|
||||
* note. Any arm that was actually run against a bonus cell is shown inline; most
|
||||
* are unrun, in which case only the capability annotation is reported.
|
||||
*/
|
||||
function renderBonus(report: Report): string[] {
|
||||
const lines = ["Bonus table — capability-asymmetric operations (outside the scored comparison):"];
|
||||
for (const row of report.bonus) {
|
||||
lines.push(` ${row.operation}`);
|
||||
lines.push(` direction: ${row.direction}; gitea-axi: ${row.giteaAxi}`);
|
||||
lines.push(` note: ${row.note}`);
|
||||
if (row.arms.length === 0) {
|
||||
lines.push(" runs: none");
|
||||
} else {
|
||||
for (const arm of row.arms) {
|
||||
lines.push(
|
||||
` runs: ${arm.arm} — ${cell(arm.costEquivalentTokens, asInteger)} cost-eq, ${cell(arm.successRate, asPercent)} pass (${arm.samples} sample(s))`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Render an aggregated report into a stable, human-readable text block. */
|
||||
export function renderReport(report: Report): string {
|
||||
const sections = [
|
||||
renderHeadline(report),
|
||||
renderCoverage(report),
|
||||
renderTiers(report),
|
||||
renderComponents(report),
|
||||
renderBonus(report),
|
||||
];
|
||||
return `${sections.map((section) => section.join("\n")).join("\n\n")}\n`;
|
||||
}
|
||||
Reference in New Issue
Block a user