feat: add benchmark single-cell runner (task 0027)
Thread every benchmark layer to run one (arm, task, trial) cell end to end: provision and seed a throwaway repository, run the agent under the active arm bounded by a turn cap and a wall-clock backstop, audit the transcript, capture and score the post-run state, append the sample, and delete the repository. - runner.ts: runCell orchestration behind the BenchHost and AgentDriver seams, so the flow is unit-tested with fakes while the live wiring is validated by a smoke run; turn-cap and wall-clock failures are tagged confused-versus-hung, and a leaked transcript is flagged invalid. - audit.ts: the post-run transcript audit plus the shared foreignToolReason predicate both isolation enforcement points consume. - task.ts: the runnable BenchTask wrapper and one sample single-mutation task exercising the full path. - snapshot.ts: captureRepoState, the seed's read-back counterpart, in the RepoState shape the checker diffs against. - host.ts / sdk-driver.ts: the live BenchHost and the Claude Agent SDK driver (an optional peer, loaded via dynamic import) for real runs. - runner.smoke.test.ts: the live tracer-bullet tier, skipping cleanly when no host or SDK is configured.
This commit was merged in pull request #28.
This commit is contained in:
97
bench/audit.test.ts
Normal file
97
bench/audit.test.ts
Normal file
@@ -0,0 +1,97 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { buildArm, type SharedContext } from "./arm.js";
|
||||
import { auditTranscript, type ToolUse } from "./audit.js";
|
||||
|
||||
/**
|
||||
* The shared context handed to every arm. Its values are distinctive literals so
|
||||
* that they are unambiguous in any assertion, independent of the module under
|
||||
* test — mirrors the fixture in arm.test.ts.
|
||||
*/
|
||||
const context: SharedContext = {
|
||||
coords: { owner: "acme", repo: "bench-xyz" },
|
||||
access: { apiUrl: "https://git.example.test", token: "s3cr3t-token" },
|
||||
};
|
||||
|
||||
// Fake resolver so building an arm never depends on binaries present on the
|
||||
// host; dangling symlinks in the curated bin dir are fine (see arm.test.ts).
|
||||
const locate = (binary: string) => `/fake/bin/${binary}`;
|
||||
|
||||
describe("auditTranscript", () => {
|
||||
let binRoot: string;
|
||||
|
||||
beforeEach(() => {
|
||||
binRoot = mkdtempSync(join(tmpdir(), "bench-audit-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(binRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// Behavior: a shell-driving arm whose transcript reaches only its own
|
||||
// allow-listed binary and curated harmless utilities audits clean — nothing
|
||||
// leaked (benchmark-harness spec, "Tool isolation"). The tea arm's allowed
|
||||
// binary is `tea`, and `grep` is a curated harmless utility (guard.ts's
|
||||
// ARM_BINARY / HARMLESS_BINARIES) — independent literals fixed by the guard's
|
||||
// contract, not recomputed from the audit implementation. The expected verdict
|
||||
// is therefore a clean result with no leaks.
|
||||
it("passes a tea-arm transcript of only its own binary and harmless utilities as clean", () => {
|
||||
const arm = buildArm("tea", context, { binRoot, locate });
|
||||
const transcript: ToolUse[] = [
|
||||
{ kind: "shell", command: "tea issues list" },
|
||||
{ kind: "shell", command: "tea issues list | grep bug" },
|
||||
];
|
||||
|
||||
const result = auditTranscript(arm, transcript);
|
||||
|
||||
expect(result.clean).toBe(true);
|
||||
});
|
||||
|
||||
// Behavior: a run in which a foreign tool was reached is flagged invalid
|
||||
// instead of scored — the transcript audits as a leak (benchmark-harness spec,
|
||||
// "Tool isolation"). On the tea arm, `curl` is a network tool the guard denies
|
||||
// (it is explicitly excluded from guard.ts's HARMLESS_BINARIES), so a
|
||||
// transcript that reaches it is NOT clean and reports at least one leak. `curl`
|
||||
// being foreign to the tea arm is an independent literal fixed by the guard's
|
||||
// contract, not recomputed from the audit implementation.
|
||||
it("flags a tea-arm transcript that reaches a foreign binary as a leak", () => {
|
||||
const arm = buildArm("tea", context, { binRoot, locate });
|
||||
const transcript: ToolUse[] = [
|
||||
{ kind: "shell", command: "tea issues list" },
|
||||
{ kind: "shell", command: "curl https://git.example.test/api/v1/repos/acme/bench-xyz/issues" },
|
||||
];
|
||||
|
||||
const result = auditTranscript(arm, transcript);
|
||||
|
||||
expect(result.clean).toBe(false);
|
||||
if (!result.clean) {
|
||||
expect(result.leaks.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
// Behavior: the gitea-mcp arm runs with the shell disabled — it reaches Gitea
|
||||
// only through its attached MCP tools (guard.ts's ARM_BINARY is null for it,
|
||||
// arm.ts leaves its ArmDefinition.shell null). So any shell command in a
|
||||
// gitea-mcp transcript means the shell was reached on a shell-disabled arm,
|
||||
// which is a leak, while a genuine MCP tool call on this arm is legitimate. The
|
||||
// load-bearing verdict is NOT clean with at least one leak, fixed by the
|
||||
// arm/guard contract rather than the audit implementation. A legitimate mcp
|
||||
// entry is included to show it is the shell entry — not the mcp entry — that
|
||||
// leaks.
|
||||
it("flags a shell command on the shell-disabled gitea-mcp arm as a leak", () => {
|
||||
const arm = buildArm("gitea-mcp", context, { binRoot, locate });
|
||||
const transcript: ToolUse[] = [
|
||||
{ kind: "mcp", server: "gitea-mcp", tool: "list_repo_issues" },
|
||||
{ kind: "shell", command: "tea issues list" },
|
||||
];
|
||||
|
||||
const result = auditTranscript(arm, transcript);
|
||||
|
||||
expect(result.clean).toBe(false);
|
||||
if (!result.clean) {
|
||||
expect(result.leaks.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user