alexion
0b0186674e
bench:run gains a --skill <path> flag, threaded into the existing BuildArmOptions.skillPath, so a skill variant can be A/B'd against the shipped SKILL.md with the same binary and harness — the mechanism used to validate the find-then-act rewrite — without mutating the shipped file.
285 lines
11 KiB
TypeScript
285 lines
11 KiB
TypeScript
// The maintainer-facing run-loop command: run one chosen benchmark cell on demand.
|
|
//
|
|
// This is the entry point the maintainer invokes to spend the token budget
|
|
// available at a given moment on exactly one `(arm, task)` cell. It parses the
|
|
// selection, resolves live host access through gitea-axi's own credential path,
|
|
// resolves the scored suite against the host's self-review support, and drives the
|
|
// run loop (run-loop.ts) — which in turn drives the single-cell runner and the
|
|
// append-only sample store built in earlier slices. No orchestration is
|
|
// reimplemented here.
|
|
//
|
|
// The command is bench-internal (bench/ is excluded from the published package)
|
|
// and is executed with a TypeScript-aware runner; see `npm run bench:run`.
|
|
//
|
|
// The argument parser (`parseRunArgs`) is the pure, unit-tested seam. The live
|
|
// wiring in `runBenchCommand` is a live boundary — it resolves real credentials
|
|
// and drives the real host and Agent SDK — so, like the seed and runner smoke
|
|
// tiers, it is validated by running it rather than by mocked unit tests.
|
|
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
import type { CliDeps } from "../src/deps.js";
|
|
import { liveBenchHost } from "./host.js";
|
|
import type { Arm } from "./result.js";
|
|
import { DEFAULT_TRIALS, REPORTING_FLOOR, runCells, type RunCellsResult } from "./run-loop.js";
|
|
import { resolveBenchAccess } from "./seed.js";
|
|
import { detectSelfReviewSupport } from "./self-review.js";
|
|
import { sdkAgentDriver } from "./sdk-driver.js";
|
|
import { createSampleStore, DEFAULT_STORE_ROOT } from "./store.js";
|
|
import { buildScoredSuite } from "./task-suite.js";
|
|
|
|
// Re-exported so this command's existing importers keep resolving it from here;
|
|
// its single source of truth is now the store, which owns the default root.
|
|
export { DEFAULT_STORE_ROOT };
|
|
|
|
/** The default turn cap a run is bounded by when not overridden. */
|
|
export const DEFAULT_TURN_CAP = 40;
|
|
|
|
/** The default wall-clock backstop (ms) a run is bounded by when not overridden. */
|
|
export const DEFAULT_WALL_CLOCK_MS = 300_000;
|
|
|
|
/** Environment variable naming the tea login the benchmark authenticates through. */
|
|
export const LOGIN_ENV = "GITEA_AXI_BENCH_LOGIN";
|
|
|
|
/** The four arms a cell may select. */
|
|
export const ARMS: readonly Arm[] = ["gitea-axi", "tea", "gitea-mcp", "raw-api"];
|
|
|
|
/** A fully-resolved cell selection and run configuration. */
|
|
export interface RunArgs {
|
|
arm: Arm;
|
|
taskId: string;
|
|
/** Trials to run this sitting; defaults to {@link DEFAULT_TRIALS}. */
|
|
trials: number;
|
|
/** The tea login the benchmark authenticates through. */
|
|
login: string;
|
|
turnCap: number;
|
|
wallClockMs: number;
|
|
storeRoot: string;
|
|
/** Optional bundled-skill override for the gitea-axi arm; defaults to the shipped SKILL.md. */
|
|
skillPath?: string;
|
|
}
|
|
|
|
/** The parse outcome: a request for help, or a resolved configuration to run. */
|
|
export type ParsedRunArgs = { help: true } | ({ help: false } & RunArgs);
|
|
|
|
/** The value-taking flags the command understands; anything else is rejected. */
|
|
const KNOWN_FLAGS = new Set([
|
|
"arm",
|
|
"task",
|
|
"trials",
|
|
"login",
|
|
"turn-cap",
|
|
"wall-clock-ms",
|
|
"store",
|
|
"skill",
|
|
]);
|
|
|
|
/** A usage error, surfaced to the maintainer with the offending detail. */
|
|
function usage(detail: string): Error {
|
|
return new Error(`${detail}\n\nUsage: bench:run --arm <arm> --task <task-id> [--login <name>] [--trials <n>]`);
|
|
}
|
|
|
|
/** Parse a flag's value as a positive integer, rejecting anything else. */
|
|
function positiveInt(value: string, flag: string): number {
|
|
if (!/^\d+$/.test(value) || Number(value) < 1) {
|
|
throw usage(`--${flag} must be a positive integer, got "${value}"`);
|
|
}
|
|
return Number(value);
|
|
}
|
|
|
|
/**
|
|
* Parse the run-loop command's argv into a resolved configuration, applying
|
|
* defaults ({@link DEFAULT_TRIALS} trials, {@link DEFAULT_TURN_CAP} turn cap,
|
|
* {@link DEFAULT_WALL_CLOCK_MS} backstop, {@link DEFAULT_STORE_ROOT} store, and the
|
|
* login from {@link LOGIN_ENV}). Throws a usage error when a required selection is
|
|
* missing or a value is malformed.
|
|
*/
|
|
export function parseRunArgs(
|
|
argv: string[],
|
|
env: Record<string, string | undefined>,
|
|
): ParsedRunArgs {
|
|
const flags = new Map<string, string>();
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const token = argv[index] as string;
|
|
if (token === "--help" || token === "-h") {
|
|
return { help: true };
|
|
}
|
|
if (!token.startsWith("--")) {
|
|
throw usage(`unexpected argument "${token}"`);
|
|
}
|
|
const equals = token.indexOf("=");
|
|
const name = equals === -1 ? token.slice(2) : token.slice(2, equals);
|
|
if (!KNOWN_FLAGS.has(name)) {
|
|
throw usage(`unknown flag "--${name}"`);
|
|
}
|
|
let value: string | undefined;
|
|
if (equals === -1) {
|
|
value = argv[index + 1];
|
|
if (value === undefined || value.startsWith("--")) {
|
|
throw usage(`flag --${name} needs a value`);
|
|
}
|
|
index += 1;
|
|
} else {
|
|
value = token.slice(equals + 1);
|
|
}
|
|
flags.set(name, value);
|
|
}
|
|
|
|
const arm = flags.get("arm");
|
|
if (arm === undefined) {
|
|
throw usage("--arm <arm> is required");
|
|
}
|
|
if (!ARMS.includes(arm as Arm)) {
|
|
throw usage(`--arm must be one of ${ARMS.join(", ")}, got "${arm}"`);
|
|
}
|
|
const taskId = flags.get("task");
|
|
if (taskId === undefined) {
|
|
throw usage("--task <task-id> is required");
|
|
}
|
|
const login = flags.get("login") ?? env[LOGIN_ENV];
|
|
if (login === undefined || login.length === 0) {
|
|
throw usage(`--login <name> is required (or set ${LOGIN_ENV})`);
|
|
}
|
|
|
|
const trials = flags.has("trials") ? positiveInt(flags.get("trials") as string, "trials") : DEFAULT_TRIALS;
|
|
const turnCap = flags.has("turn-cap")
|
|
? positiveInt(flags.get("turn-cap") as string, "turn-cap")
|
|
: DEFAULT_TURN_CAP;
|
|
const wallClockMs = flags.has("wall-clock-ms")
|
|
? positiveInt(flags.get("wall-clock-ms") as string, "wall-clock-ms")
|
|
: DEFAULT_WALL_CLOCK_MS;
|
|
|
|
return {
|
|
help: false,
|
|
arm: arm as Arm,
|
|
taskId,
|
|
trials,
|
|
login,
|
|
turnCap,
|
|
wallClockMs,
|
|
storeRoot: flags.get("store") ?? DEFAULT_STORE_ROOT,
|
|
...(flags.has("skill") ? { skillPath: flags.get("skill") } : {}),
|
|
};
|
|
}
|
|
|
|
/** The help text printed for `--help` / `-h`. */
|
|
const HELP_TEXT = `bench:run — run one benchmark cell on demand.
|
|
|
|
Runs a single (arm, task) cell for a batch of trials against the live Gitea host,
|
|
appending each scored sample to the store. Re-running a cell deepens it: new trials
|
|
append rather than overwrite, so a cell's sample size can be grown across sittings.
|
|
|
|
Usage:
|
|
npm run bench:run -- --arm <arm> --task <task-id> [options]
|
|
|
|
Required:
|
|
--arm <arm> One of: ${ARMS.join(", ")}
|
|
--task <task-id> A scored-suite task id (an unknown id prints the available ids)
|
|
|
|
Options:
|
|
--login <name> tea login to authenticate through (default: $${LOGIN_ENV})
|
|
--trials <n> Trials to run this sitting (default: ${DEFAULT_TRIALS})
|
|
--turn-cap <n> Per-run turn cap (default: ${DEFAULT_TURN_CAP})
|
|
--wall-clock-ms <n> Per-run wall-clock backstop in ms (default: ${DEFAULT_WALL_CLOCK_MS})
|
|
--store <dir> Sample store root (default: ${DEFAULT_STORE_ROOT})
|
|
--skill <path> Override the gitea-axi arm's bundled skill (default: shipped SKILL.md)
|
|
-h, --help Show this help`;
|
|
|
|
/** Render the run-loop tally into the lines printed after a sitting. */
|
|
function summarize(result: RunCellsResult, storeRoot: string): string[] {
|
|
const floorNote = result.meetsFloor
|
|
? `meets the reporting floor of ${REPORTING_FLOOR}`
|
|
: `below the reporting floor of ${REPORTING_FLOOR} — deepen this cell before reporting`;
|
|
return [
|
|
`Cell (${result.arm}, ${result.taskId}): ${result.recorded} recorded, ${result.invalid} invalid this sitting.`,
|
|
`Samples: ${result.priorSamples} → ${result.totalSamples} (${floorNote}).`,
|
|
`Store: ${storeRoot}`,
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Run one chosen cell on demand: resolve live host access, resolve the scored
|
|
* suite against the host's self-review support, select the task, and drive the run
|
|
* loop. This is the command's live boundary — it authenticates and drives the real
|
|
* host and Agent SDK — so it is validated by running it, not by mocked unit tests
|
|
* (the pure `parseRunArgs` seam is the unit-tested part). Returns a process exit
|
|
* code and prints progress and the final tally through `out`.
|
|
*/
|
|
export async function runBenchCommand(
|
|
argv: string[],
|
|
deps: CliDeps,
|
|
out: (line: string) => void,
|
|
): Promise<number> {
|
|
const parsed = parseRunArgs(argv, deps.env);
|
|
if (parsed.help) {
|
|
out(HELP_TEXT);
|
|
return 0;
|
|
}
|
|
|
|
const access = await resolveBenchAccess(deps, parsed.login);
|
|
|
|
// The two review tasks are approve/request-changes or comment reviews depending
|
|
// on what the host permits, so the suite is resolved against a live probe once
|
|
// before selecting the task (see task-suite.ts and self-review.ts).
|
|
out(`Probing self-review support on ${new URL(access.apiUrl).host}…`);
|
|
const selfReviewPermitted = await detectSelfReviewSupport(access);
|
|
const suite = buildScoredSuite({ selfReviewPermitted });
|
|
const task = suite.find((candidate) => candidate.id === parsed.taskId);
|
|
if (task === undefined) {
|
|
out(`No scored task with id "${parsed.taskId}". Available task ids:`);
|
|
for (const candidate of suite) {
|
|
out(` ${candidate.id} (${candidate.tier})`);
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
const store = createSampleStore(parsed.storeRoot);
|
|
const binRoot = mkdtempSync(join(tmpdir(), "bench-run-bin-"));
|
|
out(`Running ${parsed.trials} trial(s) of cell (${parsed.arm}, ${task.id})…`);
|
|
try {
|
|
const result = await runCells({
|
|
arm: parsed.arm,
|
|
task,
|
|
trials: parsed.trials,
|
|
access,
|
|
host: liveBenchHost(access),
|
|
// Every arm runs on the driver's single fixed model per the spec, so the
|
|
// comparison measures the tool rather than the model; the command exposes
|
|
// no per-cell model override that could break that invariant.
|
|
driver: sdkAgentDriver(),
|
|
store,
|
|
bounds: { turnCap: parsed.turnCap, wallClockMs: parsed.wallClockMs },
|
|
build: { binRoot, ...(parsed.skillPath !== undefined ? { skillPath: parsed.skillPath } : {}) },
|
|
});
|
|
for (const line of summarize(result, parsed.storeRoot)) {
|
|
out(line);
|
|
}
|
|
return 0;
|
|
} finally {
|
|
rmSync(binRoot, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
/** Entry point: parse argv, run the command, and set the process exit code. */
|
|
export async function main(): Promise<void> {
|
|
const deps: CliDeps = { env: process.env, cwd: process.cwd(), globals: {} };
|
|
try {
|
|
process.exitCode = await runBenchCommand(
|
|
process.argv.slice(2),
|
|
deps,
|
|
(line) => process.stdout.write(`${line}\n`),
|
|
);
|
|
} catch (error) {
|
|
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
// Run only when executed directly (e.g. `tsx bench/run.ts`), not when imported by
|
|
// a test. Under a TypeScript runner argv[1] is this file's own path.
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
await main();
|
|
}
|