// The maintainer-facing reporting command: render the accumulated sample store
// into the readable comparison.
//
// This is the reporting counterpart to run.ts. Where the run command spends the
// token budget on one cell, this command reads whatever samples have accumulated
// so far and prints the aggregator's comparison — headline, coverage, per-tier
// and per-token-component breakdowns, and the bonus table.
//
// Unlike run.ts, this command has no live boundary: it touches only the local
// sample store on disk (no credentials, host, or Agent SDK), so the whole command
// is deterministic and unit-tested. `parseReportArgs` is the pure argument seam.
import { pathToFileURL } from "node:url";
import type { CliDeps } from "../src/deps.js";
import { aggregate, readAllSamples, renderReport } from "./aggregate.js";
import { createSampleStore, DEFAULT_STORE_ROOT } from "./store.js";
import { buildBonusTasks, buildScoredSuite } from "./task-suite.js";
/** A fully-resolved report configuration. */
export interface ReportArgs {
/** The sample-store root to read; defaults to {@link DEFAULT_STORE_ROOT}. */
storeRoot: string;
/** Whether to render the suite/bonus variant for a self-review-permitting host. */
selfReview: boolean;
}
/** The parse outcome: a request for help, or a resolved configuration to render. */
export type ParsedReportArgs = { help: true } | ({ help: false } & ReportArgs);
/** The value-taking flags the command understands. */
const VALUE_FLAGS = new Set(["store"]);
/** The boolean flags the command understands, each with a `--no-` negation. */
const BOOLEAN_FLAGS = new Set(["self-review"]);
/** A usage error, surfaced to the maintainer with the offending detail. */
function usage(detail: string): Error {
return new Error(`${detail}\n\nUsage: bench:report [--store
] [--self-review | --no-self-review]`);
}
/**
* Parse the report command's argv into a resolved configuration, applying
* defaults ({@link DEFAULT_STORE_ROOT} store, self-review permitted). A report
* needs no required selection, so no arguments is a valid invocation. Throws a
* usage error on an unknown flag, a bare argument, a value-flag missing its
* value, or a value handed to a boolean flag.
*/
export function parseReportArgs(argv: string[]): ParsedReportArgs {
let storeRoot = DEFAULT_STORE_ROOT;
let selfReview = true;
for (let index = 0; index < argv.length; index += 1) {
const token = argv[index] as string;
if (token === "--help" || token === "-h") {
return { help: true };
}
if (!token.startsWith("--")) {
throw usage(`unexpected argument "${token}"`);
}
const equals = token.indexOf("=");
const rawName = equals === -1 ? token.slice(2) : token.slice(2, equals);
const inlineValue = equals === -1 ? undefined : token.slice(equals + 1);
// A `--no-` prefix negates a boolean flag.
const negated = rawName.startsWith("no-");
const name = negated ? rawName.slice(3) : rawName;
if (BOOLEAN_FLAGS.has(name)) {
if (inlineValue !== undefined) {
throw usage(`flag --${rawName} takes no value`);
}
selfReview = !negated;
continue;
}
if (negated || !VALUE_FLAGS.has(name)) {
throw usage(`unknown flag "--${rawName}"`);
}
let value = inlineValue;
if (value === undefined) {
value = argv[index + 1];
if (value === undefined || value.startsWith("--")) {
throw usage(`flag --${name} needs a value`);
}
index += 1;
}
if (name === "store") {
storeRoot = value;
}
}
return { help: false, storeRoot, selfReview };
}
/** The help text printed for `--help` / `-h`. */
const HELP_TEXT = `bench:report — render the accumulated benchmark samples into a comparison.
Reads whatever samples have accumulated in the store and prints the aggregator's
comparison: the cost-equivalent-token headline, coverage annotated against the
reporting floor, per-tier and per-token-component breakdowns, and the bonus table.
Incomplete coverage is annotated rather than hidden, so a half-run matrix still
renders. This command is offline — it reads only the local store, never the host.
Usage:
npm run bench:report -- [options]
Options:
--store Sample store root to read (default: ${DEFAULT_STORE_ROOT})
--self-review Render the variant for a self-review-permitting host (default)
--no-self-review Render the variant for a host that forbids self-review
-h, --help Show this help
--self-review only affects the bonus capability catalog (whether the approve /
request-changes review pair appears there or in the scored suite); the scored
coverage is identical either way. Set it to match the host the samples were run on.`;
/**
* Render the accumulated sample store into the readable comparison. This is the
* command's boundary, but — unlike the run command — it is offline: it opens the
* local store at `--store`, drains it, aggregates against the scored suite and
* bonus definitions (resolved against the `--self-review` variant), and prints the
* rendered report through `out`. No orchestration, weighting, or rendering is
* reimplemented here; it drives the `readAllSamples` / `aggregate` / `renderReport`
* seam. Returns a process exit code.
*/
export async function runReportCommand(
argv: string[],
// Unused: an offline report needs no credentials, cwd, or env. Kept for signature
// parity with the command family (runBenchCommand takes the same (argv, deps, out)).
_deps: CliDeps,
out: (line: string) => void,
): Promise {
const parsed = parseReportArgs(argv);
if (parsed.help) {
out(HELP_TEXT);
return 0;
}
const suiteOptions = { selfReviewPermitted: parsed.selfReview };
const store = createSampleStore(parsed.storeRoot);
const report = aggregate({
records: readAllSamples(store),
suite: buildScoredSuite(suiteOptions),
bonus: buildBonusTasks(suiteOptions),
});
out(renderReport(report));
return 0;
}
/** Entry point: render the report and set the process exit code. */
export async function main(): Promise {
const deps: CliDeps = { env: process.env, cwd: process.cwd(), globals: {} };
try {
process.exitCode = await runReportCommand(
process.argv.slice(2),
deps,
(line) => process.stdout.write(`${line}\n`),
);
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
process.exitCode = 1;
}
}
// Run only when executed directly (e.g. `tsx bench/report.ts`), not when imported
// by a test. Under a TypeScript runner argv[1] is this file's own path.
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
await main();
}