alexion
c6a972734e
Thread every benchmark layer to run one (arm, task, trial) cell end to end: provision and seed a throwaway repository, run the agent under the active arm bounded by a turn cap and a wall-clock backstop, audit the transcript, capture and score the post-run state, append the sample, and delete the repository. - runner.ts: runCell orchestration behind the BenchHost and AgentDriver seams, so the flow is unit-tested with fakes while the live wiring is validated by a smoke run; turn-cap and wall-clock failures are tagged confused-versus-hung, and a leaked transcript is flagged invalid. - audit.ts: the post-run transcript audit plus the shared foreignToolReason predicate both isolation enforcement points consume. - task.ts: the runnable BenchTask wrapper and one sample single-mutation task exercising the full path. - snapshot.ts: captureRepoState, the seed's read-back counterpart, in the RepoState shape the checker diffs against. - host.ts / sdk-driver.ts: the live BenchHost and the Claude Agent SDK driver (an optional peer, loaded via dynamic import) for real runs. - runner.smoke.test.ts: the live tracer-bullet tier, skipping cleanly when no host or SDK is configured.
25 lines
1.1 KiB
TypeScript
25 lines
1.1 KiB
TypeScript
// The live host adapter: the production `BenchHost` the runner drives against a
|
|
// real Gitea instance. It is a thin composition of the two live boundaries —
|
|
// seed.ts (provision, seed, delete) and snapshot.ts (capture) — bound to one set
|
|
// of host credentials. The runner depends only on the `BenchHost` seam, so this
|
|
// wiring is exercised by the smoke run rather than mocked unit tests, matching the
|
|
// seed tier.
|
|
|
|
import type { BenchHost } from "./runner.js";
|
|
import { captureRepoState } from "./snapshot.js";
|
|
import { deleteRepo, provisionRepo, seedRepo, type BenchAccess } from "./seed.js";
|
|
|
|
/**
|
|
* Build the live host bound to `access`: it provisions and seeds fresh throwaway
|
|
* repositories, captures their post-run state, and deletes them — all through the
|
|
* real Gitea API using gitea-axi's own credential discovery (see resolveBenchAccess).
|
|
*/
|
|
export function liveBenchHost(access: BenchAccess): BenchHost {
|
|
return {
|
|
provision: () => provisionRepo(access),
|
|
seed: (coords) => seedRepo(access, coords),
|
|
capture: (coords) => captureRepoState(access, coords),
|
|
delete: (coords) => deleteRepo(access, coords),
|
|
};
|
|
}
|