diff --git a/bench/sdk-driver.test.ts b/bench/sdk-driver.test.ts index 32170ed..f8521f8 100644 --- a/bench/sdk-driver.test.ts +++ b/bench/sdk-driver.test.ts @@ -1,5 +1,7 @@ +import { readdirSync, rmSync, statSync } from "node:fs"; +import path from "node:path"; import { describe, expect, it } from "vitest"; -import { sumTokens } from "./sdk-driver.js"; +import { createAgentWorkdir, sumTokens } from "./sdk-driver.js"; import type { SdkResultMessage } from "./sdk-driver.js"; describe("sumTokens", () => { @@ -87,3 +89,31 @@ describe("sumTokens", () => { }); }); }); + +describe("createAgentWorkdir", () => { + // Behavior: each agent run must operate in a fresh, empty directory located + // OUTSIDE the harness's own checkout. This isolation is what stops an agent + // that forgets an explicit `-R OWNER/NAME` from having its `gitea-axi`/`tea` + // tools silently default their target repo to the harness's own git checkout: + // a directory that is empty (no `.git`) and outside the current checkout gives + // those tools nothing local to resolve. + // + // The three assertions are independent anti-bug properties drawn directly from + // the requirement, not recomputed from the implementation: + // 1. the path exists and is a directory, + // 2. it is empty (zero entries — in particular no `.git`), + // 3. it sits outside the current working directory (relative path escapes + // upward with ".."). + it("returns a fresh, empty directory located outside the current checkout", () => { + const dir = createAgentWorkdir(); + try { + expect(statSync(dir).isDirectory()).toBe(true); + expect(readdirSync(dir)).toHaveLength(0); + expect(path.relative(process.cwd(), dir).startsWith("..")).toBe(true); + } finally { + // Safe: `dir` is a fresh throwaway temp dir we just received from + // createAgentWorkdir(); never a delete of cwd or any pre-existing path. + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/bench/sdk-driver.ts b/bench/sdk-driver.ts index 5734e26..13c2702 100644 --- a/bench/sdk-driver.ts +++ b/bench/sdk-driver.ts @@ -17,6 +17,9 @@ // recorded in the transcript, so the runner's post-run audit sees what actually // executed — a blocked attempt is realistic wasted effort, not a leak. +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import type { ArmDefinition } from "./arm.js"; import { foreignToolReason, type ToolUse } from "./audit.js"; import type { TokenComponents } from "./result.js"; @@ -104,6 +107,8 @@ interface SdkQueryOptions { abortController: AbortController; canUseTool: (toolName: string, input: Record) => Promise; settingSources: string[]; + /** The agent's shell working directory: a fresh empty dir outside any checkout. */ + cwd: string; env?: Record; mcpServers?: Record; disallowedTools?: string[]; @@ -197,30 +202,47 @@ export function sdkAgentDriver(config: SdkDriverConfig = {}): AgentDriver { return { behavior: "allow", updatedInput: toolInput }; }; - const options = buildOptions(input.arm, model, input.turnCap, controller, canUseTool); + const workdir = createAgentWorkdir(); + try { + const options = buildOptions(input.arm, model, input.turnCap, controller, canUseTool, workdir); - let result: SdkResultMessage | undefined; - for await (const message of query({ prompt: input.intent, options })) { - if (message.type === "result") { - result = message as SdkResultMessage; + let result: SdkResultMessage | undefined; + for await (const message of query({ prompt: input.intent, options })) { + if (message.type === "result") { + result = message as SdkResultMessage; + } + } + if (result === undefined) { + throw new Error("the Agent SDK produced no result message"); } - } - if (result === undefined) { - throw new Error("the Agent SDK produced no result message"); - } - return { - tokens: sumTokens(result), - turns: result.num_turns ?? 0, - imputedCostUsd: result.total_cost_usd ?? 0, - transcript, - finalReport: result.result ?? "", - stoppedByTurnCap: result.subtype === "error_max_turns", - }; + return { + tokens: sumTokens(result), + turns: result.num_turns ?? 0, + imputedCostUsd: result.total_cost_usd ?? 0, + transcript, + finalReport: result.result ?? "", + stoppedByTurnCap: result.subtype === "error_max_turns", + }; + } finally { + rmSync(workdir, { recursive: true, force: true }); + } }, }; } +/** + * Create a fresh, empty working directory for one agent run, outside any git + * checkout. The agent's shell runs here so a shell tool that defaults its target + * repository from the local checkout (gitea-axi, tea) cannot silently resolve the + * harness's own repository when the agent omits an explicit `-R`; with no ambient + * checkout the agent must target the repository named in its prompt. The caller + * deletes it when the run ends. + */ +export function createAgentWorkdir(): string { + return mkdtempSync(join(tmpdir(), "bench-agent-cwd-")); +} + /** Assemble the SDK query options for an arm's tool configuration. */ function buildOptions( arm: ArmDefinition, @@ -228,6 +250,7 @@ function buildOptions( turnCap: number, controller: AbortController, canUseTool: SdkQueryOptions["canUseTool"], + cwd: string, ): SdkQueryOptions { const options: SdkQueryOptions = { model, @@ -241,6 +264,9 @@ function buildOptions( // Start from a clean slate: no user/project settings leak tools or config // into the measured run. settingSources: [], + // Run outside any checkout so a forgotten -R cannot resolve the harness's own + // repository instead of the seeded throwaway (see createAgentWorkdir). + cwd, }; if (arm.shell !== null) { // Lead the agent's PATH with the arm's curated bin directory so only its one