// The maintainer-facing run-loop command: run one chosen benchmark cell on demand. // // This is the entry point the maintainer invokes to spend the token budget // available at a given moment on exactly one `(arm, task)` cell. It parses the // selection, resolves live host access through gitea-axi's own credential path, // resolves the scored suite against the host's self-review support, and drives the // run loop (run-loop.ts) — which in turn drives the single-cell runner and the // append-only sample store built in earlier slices. No orchestration is // reimplemented here. // // The command is bench-internal (bench/ is excluded from the published package) // and is executed with a TypeScript-aware runner; see `npm run bench:run`. // // The argument parser (`parseRunArgs`) is the pure, unit-tested seam. The live // wiring in `runBenchCommand` is a live boundary — it resolves real credentials // and drives the real host and Agent SDK — so, like the seed and runner smoke // tiers, it is validated by running it rather than by mocked unit tests. import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; import type { CliDeps } from "../src/deps.js"; import { liveBenchHost } from "./host.js"; import type { Arm } from "./result.js"; import { DEFAULT_TRIALS, REPORTING_FLOOR, runCells, type RunCellsResult } from "./run-loop.js"; import { resolveBenchAccess } from "./seed.js"; import { detectSelfReviewSupport } from "./self-review.js"; import { sdkAgentDriver } from "./sdk-driver.js"; import { createSampleStore, DEFAULT_STORE_ROOT } from "./store.js"; import { buildScoredSuite } from "./task-suite.js"; // Re-exported so this command's existing importers keep resolving it from here; // its single source of truth is now the store, which owns the default root. export { DEFAULT_STORE_ROOT }; /** The default turn cap a run is bounded by when not overridden. */ export const DEFAULT_TURN_CAP = 40; /** The default wall-clock backstop (ms) a run is bounded by when not overridden. */ export const DEFAULT_WALL_CLOCK_MS = 300_000; /** Environment variable naming the tea login the benchmark authenticates through. */ export const LOGIN_ENV = "GITEA_AXI_BENCH_LOGIN"; /** The four arms a cell may select. */ export const ARMS: readonly Arm[] = ["gitea-axi", "tea", "gitea-mcp", "raw-api"]; /** A fully-resolved cell selection and run configuration. */ export interface RunArgs { arm: Arm; taskId: string; /** Trials to run this sitting; defaults to {@link DEFAULT_TRIALS}. */ trials: number; /** The tea login the benchmark authenticates through. */ login: string; turnCap: number; wallClockMs: number; storeRoot: string; /** Optional bundled-skill override for the gitea-axi arm; defaults to the shipped SKILL.md. */ skillPath?: string; } /** The parse outcome: a request for help, or a resolved configuration to run. */ export type ParsedRunArgs = { help: true } | ({ help: false } & RunArgs); /** The value-taking flags the command understands; anything else is rejected. */ const KNOWN_FLAGS = new Set([ "arm", "task", "trials", "login", "turn-cap", "wall-clock-ms", "store", "skill", ]); /** A usage error, surfaced to the maintainer with the offending detail. */ function usage(detail: string): Error { return new Error(`${detail}\n\nUsage: bench:run --arm --task [--login ] [--trials ]`); } /** Parse a flag's value as a positive integer, rejecting anything else. */ function positiveInt(value: string, flag: string): number { if (!/^\d+$/.test(value) || Number(value) < 1) { throw usage(`--${flag} must be a positive integer, got "${value}"`); } return Number(value); } /** * Parse the run-loop command's argv into a resolved configuration, applying * defaults ({@link DEFAULT_TRIALS} trials, {@link DEFAULT_TURN_CAP} turn cap, * {@link DEFAULT_WALL_CLOCK_MS} backstop, {@link DEFAULT_STORE_ROOT} store, and the * login from {@link LOGIN_ENV}). Throws a usage error when a required selection is * missing or a value is malformed. */ export function parseRunArgs( argv: string[], env: Record, ): ParsedRunArgs { const flags = new Map(); for (let index = 0; index < argv.length; index += 1) { const token = argv[index] as string; if (token === "--help" || token === "-h") { return { help: true }; } if (!token.startsWith("--")) { throw usage(`unexpected argument "${token}"`); } const equals = token.indexOf("="); const name = equals === -1 ? token.slice(2) : token.slice(2, equals); if (!KNOWN_FLAGS.has(name)) { throw usage(`unknown flag "--${name}"`); } let value: string | undefined; if (equals === -1) { value = argv[index + 1]; if (value === undefined || value.startsWith("--")) { throw usage(`flag --${name} needs a value`); } index += 1; } else { value = token.slice(equals + 1); } flags.set(name, value); } const arm = flags.get("arm"); if (arm === undefined) { throw usage("--arm is required"); } if (!ARMS.includes(arm as Arm)) { throw usage(`--arm must be one of ${ARMS.join(", ")}, got "${arm}"`); } const taskId = flags.get("task"); if (taskId === undefined) { throw usage("--task is required"); } const login = flags.get("login") ?? env[LOGIN_ENV]; if (login === undefined || login.length === 0) { throw usage(`--login is required (or set ${LOGIN_ENV})`); } const trials = flags.has("trials") ? positiveInt(flags.get("trials") as string, "trials") : DEFAULT_TRIALS; const turnCap = flags.has("turn-cap") ? positiveInt(flags.get("turn-cap") as string, "turn-cap") : DEFAULT_TURN_CAP; const wallClockMs = flags.has("wall-clock-ms") ? positiveInt(flags.get("wall-clock-ms") as string, "wall-clock-ms") : DEFAULT_WALL_CLOCK_MS; return { help: false, arm: arm as Arm, taskId, trials, login, turnCap, wallClockMs, storeRoot: flags.get("store") ?? DEFAULT_STORE_ROOT, ...(flags.has("skill") ? { skillPath: flags.get("skill") } : {}), }; } /** The help text printed for `--help` / `-h`. */ const HELP_TEXT = `bench:run — run one benchmark cell on demand. Runs a single (arm, task) cell for a batch of trials against the live Gitea host, appending each scored sample to the store. Re-running a cell deepens it: new trials append rather than overwrite, so a cell's sample size can be grown across sittings. Usage: npm run bench:run -- --arm --task [options] Required: --arm One of: ${ARMS.join(", ")} --task A scored-suite task id (an unknown id prints the available ids) Options: --login tea login to authenticate through (default: $${LOGIN_ENV}) --trials Trials to run this sitting (default: ${DEFAULT_TRIALS}) --turn-cap Per-run turn cap (default: ${DEFAULT_TURN_CAP}) --wall-clock-ms Per-run wall-clock backstop in ms (default: ${DEFAULT_WALL_CLOCK_MS}) --store Sample store root (default: ${DEFAULT_STORE_ROOT}) --skill Override the gitea-axi arm's bundled skill (default: shipped SKILL.md) -h, --help Show this help`; /** Render the run-loop tally into the lines printed after a sitting. */ function summarize(result: RunCellsResult, storeRoot: string): string[] { const floorNote = result.meetsFloor ? `meets the reporting floor of ${REPORTING_FLOOR}` : `below the reporting floor of ${REPORTING_FLOOR} — deepen this cell before reporting`; return [ `Cell (${result.arm}, ${result.taskId}): ${result.recorded} recorded, ${result.invalid} invalid this sitting.`, `Samples: ${result.priorSamples} → ${result.totalSamples} (${floorNote}).`, `Store: ${storeRoot}`, ]; } /** * Run one chosen cell on demand: resolve live host access, resolve the scored * suite against the host's self-review support, select the task, and drive the run * loop. This is the command's live boundary — it authenticates and drives the real * host and Agent SDK — so it is validated by running it, not by mocked unit tests * (the pure `parseRunArgs` seam is the unit-tested part). Returns a process exit * code and prints progress and the final tally through `out`. */ export async function runBenchCommand( argv: string[], deps: CliDeps, out: (line: string) => void, ): Promise { const parsed = parseRunArgs(argv, deps.env); if (parsed.help) { out(HELP_TEXT); return 0; } const access = await resolveBenchAccess(deps, parsed.login); // The two review tasks are approve/request-changes or comment reviews depending // on what the host permits, so the suite is resolved against a live probe once // before selecting the task (see task-suite.ts and self-review.ts). out(`Probing self-review support on ${new URL(access.apiUrl).host}…`); const selfReviewPermitted = await detectSelfReviewSupport(access); const suite = buildScoredSuite({ selfReviewPermitted }); const task = suite.find((candidate) => candidate.id === parsed.taskId); if (task === undefined) { out(`No scored task with id "${parsed.taskId}". Available task ids:`); for (const candidate of suite) { out(` ${candidate.id} (${candidate.tier})`); } return 1; } const store = createSampleStore(parsed.storeRoot); const binRoot = mkdtempSync(join(tmpdir(), "bench-run-bin-")); out(`Running ${parsed.trials} trial(s) of cell (${parsed.arm}, ${task.id})…`); try { const result = await runCells({ arm: parsed.arm, task, trials: parsed.trials, access, host: liveBenchHost(access), // Every arm runs on the driver's single fixed model per the spec, so the // comparison measures the tool rather than the model; the command exposes // no per-cell model override that could break that invariant. driver: sdkAgentDriver(), store, bounds: { turnCap: parsed.turnCap, wallClockMs: parsed.wallClockMs }, build: { binRoot, ...(parsed.skillPath !== undefined ? { skillPath: parsed.skillPath } : {}) }, }); for (const line of summarize(result, parsed.storeRoot)) { out(line); } return 0; } finally { rmSync(binRoot, { recursive: true, force: true }); } } /** Entry point: parse argv, run the command, and set the process exit code. */ export async function main(): Promise { const deps: CliDeps = { env: process.env, cwd: process.cwd(), globals: {} }; try { process.exitCode = await runBenchCommand( process.argv.slice(2), deps, (line) => process.stdout.write(`${line}\n`), ); } catch (error) { process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); process.exitCode = 1; } } // Run only when executed directly (e.g. `tsx bench/run.ts`), not when imported by // a test. Under a TypeScript runner argv[1] is this file's own path. if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { await main(); }