alexion
7216d3cc31
Add bench/aggregate.ts: the pure aggregator seam that rolls the accumulated sample store into a readable comparison. aggregate produces a headline table (one row per arm, cost-equivalent tokens as the headline plus raw tokens, turns, duration, success rate, coverage, and imputed cost as a de-emphasized secondary column), per-tier and per-token-component breakdowns, and the separate bonus table; renderReport renders it as stable text. Cost-equivalent tokens are weighted at render time from the four retained components per ADR 0014, and incomplete coverage is annotated rather than hidden. Unit-tested against synthetic append-only sample stores.
554 lines
22 KiB
TypeScript
554 lines
22 KiB
TypeScript
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
import {
|
|
aggregate,
|
|
costEquivalentTokens,
|
|
readAllSamples,
|
|
renderReport,
|
|
} from "./aggregate.js";
|
|
import type { ResultRecord, TokenComponents } from "./result.js";
|
|
import { createSampleStore } from "./store.js";
|
|
|
|
/** A minimal ResultRecord for the gitea-axi arm, overridable per sample. */
|
|
function record(overrides: Partial<ResultRecord> = {}): ResultRecord {
|
|
return {
|
|
arm: "gitea-axi",
|
|
taskId: "t1",
|
|
tier: "read",
|
|
trial: 1,
|
|
timestamp: "2026-07-16T00:00:00Z",
|
|
tokens: { freshInput: 0, cacheCreation: 0, cacheRead: 0, output: 0 },
|
|
turns: 0,
|
|
durationMs: 0,
|
|
imputedCostUsd: 0,
|
|
outcome: { pass: true },
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
describe("costEquivalentTokens", () => {
|
|
// Behavior: cost-equivalent tokens weight a run's four retained token
|
|
// components by Anthropic's published API pricing ratios — fresh input 1x,
|
|
// cache-write 1.25x, cache-read 0.1x, output 5x (ADR 0014). The expected
|
|
// total is derived BY HAND from those ratios, not recomputed the way the
|
|
// code would, so it independently pins the metric.
|
|
//
|
|
// Distinct components are chosen so each of the four weighted products is a
|
|
// different number and no two raw components share a value; a wrong weight on
|
|
// any single component therefore cannot be masked by another:
|
|
// freshInput 400 * 1 = 400
|
|
// cacheCreation 800 * 1.25 = 1000
|
|
// cacheRead 2000 * 0.1 = 200
|
|
// output 600 * 5 = 3000
|
|
// total = 4600
|
|
it("weights the four retained token components by the documented pricing ratios", () => {
|
|
const tokens: TokenComponents = {
|
|
freshInput: 400,
|
|
cacheCreation: 800,
|
|
cacheRead: 2000,
|
|
output: 600,
|
|
};
|
|
|
|
expect(costEquivalentTokens(tokens)).toBe(4600);
|
|
});
|
|
});
|
|
|
|
describe("aggregate", () => {
|
|
// Behavior: the headline view yields one row per arm whose per-run metrics —
|
|
// cost-equivalent tokens (the headline), raw tokens, turns, duration, and
|
|
// imputed cost — are the MEANS across that arm's samples, and whose success
|
|
// rate is the fraction of its runs that passed. Here the gitea-axi arm has
|
|
// two samples on task t1 (one pass, one fail). Every expected number below is
|
|
// derived BY HAND from the two samples and the ADR 0014 pricing ratios
|
|
// (fresh input 1x, cache-write 1.25x, cache-read 0.1x, output 5x), so it
|
|
// pins the metric independently of how aggregate computes it:
|
|
// cost-equiv A = 100*1 + 20*5 = 200; B = 200*1 + 40*5 = 400; mean = 300
|
|
// raw tokens A = 120; B = 240; mean = 180
|
|
// turns (5 + 7) / 2 = 6
|
|
// durationMs (1000 + 3000) / 2 = 2000
|
|
// successRate 1 pass of 2 = 0.5
|
|
// imputedCost (0.02 + 0.06) / 2 = 0.04
|
|
it("yields one headline row per arm with per-run means and the passing fraction as success rate", () => {
|
|
const sampleA = record({
|
|
trial: 1,
|
|
tokens: { freshInput: 100, cacheCreation: 0, cacheRead: 0, output: 20 },
|
|
turns: 5,
|
|
durationMs: 1000,
|
|
imputedCostUsd: 0.02,
|
|
outcome: { pass: true },
|
|
});
|
|
const sampleB = record({
|
|
trial: 2,
|
|
tokens: { freshInput: 200, cacheCreation: 0, cacheRead: 0, output: 40 },
|
|
turns: 7,
|
|
durationMs: 3000,
|
|
imputedCostUsd: 0.06,
|
|
outcome: { pass: false, failure: "incorrect" },
|
|
});
|
|
|
|
const report = aggregate({
|
|
records: [sampleA, sampleB],
|
|
suite: [{ id: "t1", tier: "read" }],
|
|
bonus: [],
|
|
});
|
|
|
|
const row = report.headline.find((r) => r.arm === "gitea-axi");
|
|
if (!row) throw new Error("expected a gitea-axi headline row");
|
|
|
|
expect(row.samples).toBe(2);
|
|
expect(row.costEquivalentTokens).toBeCloseTo(300);
|
|
expect(row.rawTokens).toBeCloseTo(180);
|
|
expect(row.turns).toBeCloseTo(6);
|
|
expect(row.durationMs).toBeCloseTo(2000);
|
|
expect(row.successRate).toBeCloseTo(0.5);
|
|
expect(row.imputedCostUsd).toBeCloseTo(0.04);
|
|
});
|
|
|
|
// Behavior: the per-tier breakdown is rendered from the same records — grouped
|
|
// by tier, then by arm — reporting each tier-arm's cost-equivalent-token mean
|
|
// and passing fraction. Three gitea-axi samples span two tiers: two on a read
|
|
// task r1 (one pass, one fail) and one on a single-mutation task m1 (pass).
|
|
// Every expected number is derived BY HAND from those samples and the ADR 0014
|
|
// weights (fresh input 1x, output 5x; no cache here), independent of how
|
|
// aggregate computes them:
|
|
// read gitea-axi: cost A = 100 + 20*5 = 200; B = 200 + 40*5 = 400;
|
|
// mean = 300; success 1 of 2 = 0.5; samples 2
|
|
// single-mutation gitea-axi: cost = 50 + 10*5 = 100; success 1 of 1 = 1;
|
|
// samples 1
|
|
// The tier order is the fixed reporting order, an independent literal.
|
|
it("breaks records down per tier and per arm with cost-equivalent-token means and success rates", () => {
|
|
const r1a = record({
|
|
taskId: "r1",
|
|
tier: "read",
|
|
trial: 1,
|
|
tokens: { freshInput: 100, cacheCreation: 0, cacheRead: 0, output: 20 },
|
|
outcome: { pass: true },
|
|
});
|
|
const r1b = record({
|
|
taskId: "r1",
|
|
tier: "read",
|
|
trial: 2,
|
|
tokens: { freshInput: 200, cacheCreation: 0, cacheRead: 0, output: 40 },
|
|
outcome: { pass: false, failure: "incorrect" },
|
|
});
|
|
const m1c = record({
|
|
taskId: "m1",
|
|
tier: "single-mutation",
|
|
trial: 1,
|
|
tokens: { freshInput: 50, cacheCreation: 0, cacheRead: 0, output: 10 },
|
|
outcome: { pass: true },
|
|
});
|
|
|
|
const report = aggregate({
|
|
records: [r1a, r1b, m1c],
|
|
suite: [
|
|
{ id: "r1", tier: "read" },
|
|
{ id: "m1", tier: "single-mutation" },
|
|
],
|
|
bonus: [],
|
|
});
|
|
|
|
// One breakdown per tier, in the fixed reporting order.
|
|
expect(report.tiers.map((t) => t.tier)).toEqual([
|
|
"read",
|
|
"single-mutation",
|
|
"find-then-act",
|
|
"multi-step",
|
|
]);
|
|
|
|
const readAxi = report.tiers
|
|
.find((t) => t.tier === "read")
|
|
?.arms.find((a) => a.arm === "gitea-axi");
|
|
if (!readAxi) throw new Error("expected a read/gitea-axi tier-arm");
|
|
expect(readAxi.samples).toBe(2);
|
|
expect(readAxi.costEquivalentTokens).toBeCloseTo(300);
|
|
expect(readAxi.successRate).toBeCloseTo(0.5);
|
|
|
|
const mutAxi = report.tiers
|
|
.find((t) => t.tier === "single-mutation")
|
|
?.arms.find((a) => a.arm === "gitea-axi");
|
|
if (!mutAxi) throw new Error("expected a single-mutation/gitea-axi tier-arm");
|
|
expect(mutAxi.samples).toBe(1);
|
|
expect(mutAxi.costEquivalentTokens).toBeCloseTo(100);
|
|
expect(mutAxi.successRate).toBeCloseTo(1);
|
|
});
|
|
|
|
// Behavior: the per-token-component breakdown is rendered from the same
|
|
// records — per arm, the per-run MEAN of each of the four raw token
|
|
// components — so a reader can see what drives an arm's cost. The gitea-axi
|
|
// arm has two samples; an arm with no samples reports null for every
|
|
// component. Each mean is derived BY HAND from the two samples, independent
|
|
// of how aggregate computes it:
|
|
// freshInput (100 + 200) / 2 = 150
|
|
// cacheCreation (40 + 60) / 2 = 50
|
|
// cacheRead (1000 + 3000)/2 = 2000
|
|
// output (20 + 40) / 2 = 30
|
|
it("breaks records down per arm into the per-run mean of each token component", () => {
|
|
const sampleA = record({
|
|
trial: 1,
|
|
tokens: { freshInput: 100, cacheCreation: 40, cacheRead: 1000, output: 20 },
|
|
});
|
|
const sampleB = record({
|
|
trial: 2,
|
|
tokens: { freshInput: 200, cacheCreation: 60, cacheRead: 3000, output: 40 },
|
|
});
|
|
|
|
const report = aggregate({
|
|
records: [sampleA, sampleB],
|
|
suite: [{ id: "t1", tier: "read" }],
|
|
bonus: [],
|
|
});
|
|
|
|
const axi = report.components.find((c) => c.arm === "gitea-axi");
|
|
if (!axi) throw new Error("expected a gitea-axi component breakdown");
|
|
expect(axi.freshInput).toBeCloseTo(150);
|
|
expect(axi.cacheCreation).toBeCloseTo(50);
|
|
expect(axi.cacheRead).toBeCloseTo(2000);
|
|
expect(axi.output).toBeCloseTo(30);
|
|
|
|
// An arm with no samples reports null for every component.
|
|
const tea = report.components.find((c) => c.arm === "tea");
|
|
if (!tea) throw new Error("expected a tea component breakdown");
|
|
expect(tea.freshInput).toBeNull();
|
|
expect(tea.cacheCreation).toBeNull();
|
|
expect(tea.cacheRead).toBeNull();
|
|
expect(tea.output).toBeNull();
|
|
});
|
|
|
|
// Behavior: the separate bonus table is rendered from the same records. It
|
|
// emits one row per supplied bonus definition, in the given order, carrying
|
|
// that operation's capability metadata (operation text, direction, note, and
|
|
// gitea-axi's own applicability), plus — for any arm that actually has
|
|
// samples for that bonus cell — the per-arm run metrics. Bonus cells are
|
|
// usually unrun, so a cell's arms list is empty unless records exist for it.
|
|
// Here bonus-x has one tea sample and bonus-y has none. The one derived
|
|
// number is by hand from that sample and the ADR 0014 weights (fresh input
|
|
// 1x, output 5x): cost = 100 + 20*5 = 200.
|
|
it("emits one bonus row per definition with its capability metadata and per-arm samples only where run", () => {
|
|
const teaSample = record({
|
|
arm: "tea",
|
|
taskId: "bonus-x",
|
|
tier: "read",
|
|
trial: 1,
|
|
tokens: { freshInput: 100, cacheCreation: 0, cacheRead: 0, output: 20 },
|
|
outcome: { pass: true },
|
|
});
|
|
|
|
const report = aggregate({
|
|
records: [teaSample],
|
|
suite: [],
|
|
bonus: [
|
|
{
|
|
id: "bonus-x",
|
|
operation: "Do the X thing",
|
|
direction: "gitea-axi-advantage",
|
|
note: "only gitea-axi does X ergonomically",
|
|
giteaAxi: "applicable",
|
|
},
|
|
{
|
|
id: "bonus-y",
|
|
operation: "Do the Y thing",
|
|
direction: "gitea-axi-not-applicable",
|
|
note: "Y is outside gitea-axi's surface",
|
|
giteaAxi: "not-applicable",
|
|
},
|
|
],
|
|
});
|
|
|
|
// One row per definition, in the given order.
|
|
expect(report.bonus.map((b) => b.id)).toEqual(["bonus-x", "bonus-y"]);
|
|
|
|
// bonus-x carries its capability metadata and the tea cell's metrics.
|
|
const bx = report.bonus.find((b) => b.id === "bonus-x");
|
|
if (!bx) throw new Error("expected a bonus-x row");
|
|
expect(bx.operation).toBe("Do the X thing");
|
|
expect(bx.direction).toBe("gitea-axi-advantage");
|
|
expect(bx.note).toBe("only gitea-axi does X ergonomically");
|
|
expect(bx.giteaAxi).toBe("applicable");
|
|
|
|
const bxTea = bx.arms.find((a) => a.arm === "tea");
|
|
if (!bxTea) throw new Error("expected a tea entry on bonus-x");
|
|
expect(bx.arms).toHaveLength(1);
|
|
expect(bxTea.samples).toBe(1);
|
|
expect(bxTea.costEquivalentTokens).toBeCloseTo(200);
|
|
expect(bxTea.successRate).toBeCloseTo(1);
|
|
|
|
// bonus-y was never run, so it carries metadata but no arm metrics.
|
|
const by = report.bonus.find((b) => b.id === "bonus-y");
|
|
if (!by) throw new Error("expected a bonus-y row");
|
|
expect(by.giteaAxi).toBe("not-applicable");
|
|
expect(by.direction).toBe("gitea-axi-not-applicable");
|
|
expect(by.arms).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe("renderReport", () => {
|
|
// Behavior: the rendered headline table shows one row per arm, presents
|
|
// cost-equivalent tokens as the headline metric column and imputed cost as a
|
|
// de-emphasized secondary column, and marks an arm with no samples with the
|
|
// "—" placeholder. Two gitea-axi samples on t1 give a cost-equivalent mean of
|
|
// (200 + 400)/2 = 300 and an imputed-cost mean of (0.02 + 0.06)/2 = 0.04,
|
|
// both derived BY HAND, independent of how the report is rendered. Assertions
|
|
// target the gitea-axi row's own line and the presence of labels/values —
|
|
// never fixed column widths — so they survive a layout refactor.
|
|
it("renders one row per arm with cost-equivalent tokens as the headline and imputed cost as a secondary column", () => {
|
|
const sampleA = record({
|
|
trial: 1,
|
|
tokens: { freshInput: 100, cacheCreation: 0, cacheRead: 0, output: 20 },
|
|
turns: 5,
|
|
durationMs: 1000,
|
|
imputedCostUsd: 0.02,
|
|
outcome: { pass: true },
|
|
});
|
|
const sampleB = record({
|
|
trial: 2,
|
|
tokens: { freshInput: 200, cacheCreation: 0, cacheRead: 0, output: 40 },
|
|
turns: 7,
|
|
durationMs: 3000,
|
|
imputedCostUsd: 0.06,
|
|
outcome: { pass: false, failure: "incorrect" },
|
|
});
|
|
|
|
const output = renderReport(
|
|
aggregate({
|
|
records: [sampleA, sampleB],
|
|
suite: [{ id: "t1", tier: "read" }],
|
|
bonus: [],
|
|
}),
|
|
);
|
|
|
|
// Every arm gets a row.
|
|
expect(output).toContain("gitea-axi");
|
|
expect(output).toContain("tea");
|
|
expect(output).toContain("gitea-mcp");
|
|
expect(output).toContain("raw-api");
|
|
|
|
// The headline metric and the secondary column are labelled.
|
|
expect(output.toLowerCase()).toContain("cost-equivalent");
|
|
expect(output.toLowerCase()).toContain("imputed");
|
|
|
|
// The gitea-axi row carries its cost-equivalent mean (300) and imputed
|
|
// mean (0.04) on its own line.
|
|
const axiLine = output.split("\n").find((l) => l.includes("gitea-axi"));
|
|
if (!axiLine) throw new Error("expected a gitea-axi row line");
|
|
expect(axiLine).toContain("300");
|
|
expect(axiLine).toContain("0.04");
|
|
|
|
// An arm with no samples shows the em-dash placeholder on its row. The
|
|
// "tea" arm token is matched at a word boundary so the substring inside
|
|
// "gitea-axi"/"gitea-mcp" cannot be mistaken for the tea row.
|
|
const teaLine = output
|
|
.split("\n")
|
|
.find((l) => /(^|[^a-z-])tea([^a-z-]|$)/.test(l));
|
|
if (!teaLine) throw new Error("expected a tea row line");
|
|
expect(teaLine).toContain("—");
|
|
});
|
|
|
|
// Behavior: a partial matrix renders without error and incomplete coverage is
|
|
// annotated rather than hidden or treated as complete. With the default
|
|
// reporting floor of 3, a cell is covered at >=3 samples, partial at 1-2, and
|
|
// missing at 0. gitea-axi has one covered / one partial / one missing cell,
|
|
// so it is incomplete; tea has all three cells covered, so it is complete.
|
|
// The coverage counts (1 partial, 1 missing for gitea-axi) are derived BY
|
|
// HAND from the sample layout, independent of how the report is rendered.
|
|
// Lines are located by content, not index, so the assertions survive a
|
|
// layout refactor.
|
|
it("renders a partial matrix without throwing and annotates incomplete coverage rather than hiding it", () => {
|
|
const read = (taskId: string, trial: number) =>
|
|
record({ taskId, tier: "read", trial, outcome: { pass: true } });
|
|
|
|
const records: ResultRecord[] = [
|
|
// gitea-axi: t1 covered (3), t2 partial (2), t3 missing (0).
|
|
read("t1", 1),
|
|
read("t1", 2),
|
|
read("t1", 3),
|
|
read("t2", 1),
|
|
read("t2", 2),
|
|
// tea: all three tasks covered (3 each).
|
|
{ ...read("t1", 1), arm: "tea" },
|
|
{ ...read("t1", 2), arm: "tea" },
|
|
{ ...read("t1", 3), arm: "tea" },
|
|
{ ...read("t2", 1), arm: "tea" },
|
|
{ ...read("t2", 2), arm: "tea" },
|
|
{ ...read("t2", 3), arm: "tea" },
|
|
{ ...read("t3", 1), arm: "tea" },
|
|
{ ...read("t3", 2), arm: "tea" },
|
|
{ ...read("t3", 3), arm: "tea" },
|
|
];
|
|
|
|
const input = {
|
|
records,
|
|
suite: [
|
|
{ id: "t1", tier: "read" as const },
|
|
{ id: "t2", tier: "read" as const },
|
|
{ id: "t3", tier: "read" as const },
|
|
],
|
|
bonus: [],
|
|
};
|
|
|
|
// A partial matrix renders without error.
|
|
expect(() => renderReport(aggregate(input))).not.toThrow();
|
|
|
|
const output = renderReport(aggregate(input));
|
|
|
|
// The coverage annotation names the reporting floor it is measured against.
|
|
expect(output).toContain("reporting floor of 3");
|
|
|
|
// gitea-axi is incomplete: 1 partial, 1 missing, flagged incomplete.
|
|
const axiLine = output
|
|
.split("\n")
|
|
.find((l) => /\bgitea-axi\b/.test(l) && l.includes("partial"));
|
|
if (!axiLine) throw new Error("expected a gitea-axi coverage annotation");
|
|
expect(axiLine).toContain("1 partial");
|
|
expect(axiLine).toContain("1 missing");
|
|
expect(axiLine).toContain("incomplete");
|
|
|
|
// tea is complete and is not labelled incomplete.
|
|
const teaLine = output
|
|
.split("\n")
|
|
.find((l) => /(^|[^a-z-])tea([^a-z-]|$)/.test(l) && l.includes("complete"));
|
|
if (!teaLine) throw new Error("expected a tea coverage annotation");
|
|
expect(teaLine).toContain("complete");
|
|
expect(teaLine).not.toContain("incomplete");
|
|
});
|
|
|
|
// Behavior: the per-tier breakdown, the per-token-component breakdown, and the
|
|
// separate bonus table are all rendered from the same records — each as its
|
|
// own section surfacing its content. A single gitea-axi sample on a read task
|
|
// fixes the component means (one sample, so each mean equals its raw value):
|
|
// fresh input 300, cache write 80, cache read 1000, output 60 — derived BY
|
|
// HAND, independent of how the report renders. Assertions are presence-based
|
|
// (section headers and content strings), not column layout, so they survive a
|
|
// rendering refactor.
|
|
it("renders the per-tier, per-token-component, and bonus sections from the same records", () => {
|
|
const output = renderReport(
|
|
aggregate({
|
|
records: [
|
|
record({
|
|
taskId: "r1",
|
|
tier: "read",
|
|
trial: 1,
|
|
tokens: {
|
|
freshInput: 300,
|
|
cacheCreation: 80,
|
|
cacheRead: 1000,
|
|
output: 60,
|
|
},
|
|
outcome: { pass: true },
|
|
}),
|
|
],
|
|
suite: [
|
|
{ id: "r1", tier: "read" },
|
|
{ id: "m1", tier: "single-mutation" },
|
|
],
|
|
bonus: [
|
|
{
|
|
id: "bonus-x",
|
|
operation: "Do the X thing",
|
|
direction: "gitea-axi-advantage",
|
|
note: "only gitea-axi does X",
|
|
giteaAxi: "applicable",
|
|
},
|
|
],
|
|
}),
|
|
);
|
|
|
|
const lower = output.toLowerCase();
|
|
|
|
// Per-tier section: a header naming tiers, and every tier surfaced.
|
|
expect(lower).toContain("tier");
|
|
expect(output).toContain("read");
|
|
expect(output).toContain("single-mutation");
|
|
expect(output).toContain("find-then-act");
|
|
expect(output).toContain("multi-step");
|
|
|
|
// Per-token-component section: a header, the four component labels, and
|
|
// gitea-axi's component means.
|
|
expect(lower).toContain("component");
|
|
expect(lower).toContain("fresh input");
|
|
expect(lower).toContain("cache write");
|
|
expect(lower).toContain("cache read");
|
|
expect(lower).toContain("output");
|
|
expect(output).toContain("300");
|
|
expect(output).toContain("80");
|
|
expect(output).toContain("1000");
|
|
expect(output).toContain("60");
|
|
|
|
// Bonus section: a header, the operation text, and gitea-axi's applicability.
|
|
expect(lower).toContain("bonus");
|
|
expect(output).toContain("Do the X thing");
|
|
expect(output).toContain("applicable");
|
|
});
|
|
});
|
|
|
|
describe("renderReport over an append-only sample store", () => {
|
|
let rootA: string;
|
|
let rootB: string;
|
|
|
|
beforeEach(() => {
|
|
rootA = mkdtempSync(join(tmpdir(), "bench-aggregate-"));
|
|
rootB = mkdtempSync(join(tmpdir(), "bench-aggregate-"));
|
|
});
|
|
|
|
afterEach(() => {
|
|
rmSync(rootA, { recursive: true, force: true });
|
|
rmSync(rootB, { recursive: true, force: true });
|
|
});
|
|
|
|
// Behavior: rendering is stable — the rendered report is a function of the
|
|
// accumulated set of samples only, so the order in which samples were
|
|
// appended to the store must not change the output. This is a stability
|
|
// property a correct implementation already satisfies, exercised end to end
|
|
// through the real append-only store. The same records are appended to two
|
|
// independent stores in genuinely different orders (one reversed and with
|
|
// cells interleaved differently); the two rendered reports must be identical.
|
|
it("renders identically regardless of the order samples were appended", () => {
|
|
// A set spanning two arms and two tasks; the gitea-axi/t1 cell reaches the
|
|
// reporting floor of three so the headline and coverage sections are
|
|
// non-trivial rather than all placeholders.
|
|
const records: ResultRecord[] = [
|
|
record({ arm: "gitea-axi", taskId: "t1", tier: "read", trial: 1 }),
|
|
record({ arm: "gitea-axi", taskId: "t1", tier: "read", trial: 2 }),
|
|
record({ arm: "gitea-axi", taskId: "t1", tier: "read", trial: 3 }),
|
|
record({ arm: "gitea-axi", taskId: "t2", tier: "single-mutation", trial: 1 }),
|
|
record({ arm: "tea", taskId: "t1", tier: "read", trial: 1 }),
|
|
record({ arm: "tea", taskId: "t2", tier: "single-mutation", trial: 1 }),
|
|
record({ arm: "tea", taskId: "t2", tier: "single-mutation", trial: 2 }),
|
|
];
|
|
|
|
// Two genuinely different append orders over the same set.
|
|
const orderA = records;
|
|
const orderB = [...records].reverse();
|
|
// The two orders truly differ, so the stability claim is not vacuous.
|
|
expect(orderB).not.toEqual(orderA);
|
|
|
|
const storeA = createSampleStore(rootA);
|
|
for (const r of orderA) storeA.append(r);
|
|
|
|
const storeB = createSampleStore(rootB);
|
|
for (const r of orderB) storeB.append(r);
|
|
|
|
const suite = [
|
|
{ id: "t1", tier: "read" as const },
|
|
{ id: "t2", tier: "single-mutation" as const },
|
|
];
|
|
|
|
const a = renderReport(
|
|
aggregate({ records: readAllSamples(storeA), suite, bonus: [] }),
|
|
);
|
|
const b = renderReport(
|
|
aggregate({ records: readAllSamples(storeB), suite, bonus: [] }),
|
|
);
|
|
|
|
// Append order does not affect the rendered report.
|
|
expect(a).toBe(b);
|
|
|
|
// Non-vacuity: the report actually rendered real headline content.
|
|
expect(a.length).toBeGreaterThan(0);
|
|
expect(a).toContain("gitea-axi");
|
|
expect(a.toLowerCase()).toContain("cost-equivalent");
|
|
});
|
|
});
|