fix: run the bench agent in a neutral working directory
Some checks failed
CI / test (pull_request) Failing after 52s

The SDK driver ran the agent with no explicit cwd, so its shell inherited
the harness's own checkout. When the agent omitted `-R OWNER/NAME`, the
gitea-axi (and tea) CLI defaulted the repository from that local checkout —
silently resolving the harness repo instead of the seeded throwaway — and
returned a plausible but wrong result (e.g. `count: 0 open of 0 total` for a
repo with no issues). This contaminated read-tier scoring for the checkout-
defaulting arms and was surfaced by the newly persisted read reports.

Give each run a fresh, empty working directory outside any checkout, so a
forgotten `-R` errors instead of hitting the wrong repository, and delete it
when the run ends.
This commit is contained in:
2026-07-17 10:46:45 -04:00
parent 1166a48130
commit a6ab749211
2 changed files with 74 additions and 18 deletions

View File

@@ -1,5 +1,7 @@
import { readdirSync, rmSync, statSync } from "node:fs";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { sumTokens } from "./sdk-driver.js";
import { createAgentWorkdir, sumTokens } from "./sdk-driver.js";
import type { SdkResultMessage } from "./sdk-driver.js";
describe("sumTokens", () => {
@@ -87,3 +89,31 @@ describe("sumTokens", () => {
});
});
});
describe("createAgentWorkdir", () => {
// Behavior: each agent run must operate in a fresh, empty directory located
// OUTSIDE the harness's own checkout. This isolation is what stops an agent
// that forgets an explicit `-R OWNER/NAME` from having its `gitea-axi`/`tea`
// tools silently default their target repo to the harness's own git checkout:
// a directory that is empty (no `.git`) and outside the current checkout gives
// those tools nothing local to resolve.
//
// The three assertions are independent anti-bug properties drawn directly from
// the requirement, not recomputed from the implementation:
// 1. the path exists and is a directory,
// 2. it is empty (zero entries — in particular no `.git`),
// 3. it sits outside the current working directory (relative path escapes
// upward with "..").
it("returns a fresh, empty directory located outside the current checkout", () => {
const dir = createAgentWorkdir();
try {
expect(statSync(dir).isDirectory()).toBe(true);
expect(readdirSync(dir)).toHaveLength(0);
expect(path.relative(process.cwd(), dir).startsWith("..")).toBe(true);
} finally {
// Safe: `dir` is a fresh throwaway temp dir we just received from
// createAgentWorkdir(); never a delete of cwd or any pre-existing path.
rmSync(dir, { recursive: true, force: true });
}
});
});

View File

@@ -17,6 +17,9 @@
// recorded in the transcript, so the runner's post-run audit sees what actually
// executed — a blocked attempt is realistic wasted effort, not a leak.
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { ArmDefinition } from "./arm.js";
import { foreignToolReason, type ToolUse } from "./audit.js";
import type { TokenComponents } from "./result.js";
@@ -104,6 +107,8 @@ interface SdkQueryOptions {
abortController: AbortController;
canUseTool: (toolName: string, input: Record<string, unknown>) => Promise<SdkPermissionResult>;
settingSources: string[];
/** The agent's shell working directory: a fresh empty dir outside any checkout. */
cwd: string;
env?: Record<string, string | undefined>;
mcpServers?: Record<string, SdkStdioServer>;
disallowedTools?: string[];
@@ -197,7 +202,9 @@ export function sdkAgentDriver(config: SdkDriverConfig = {}): AgentDriver {
return { behavior: "allow", updatedInput: toolInput };
};
const options = buildOptions(input.arm, model, input.turnCap, controller, canUseTool);
const workdir = createAgentWorkdir();
try {
const options = buildOptions(input.arm, model, input.turnCap, controller, canUseTool, workdir);
let result: SdkResultMessage | undefined;
for await (const message of query({ prompt: input.intent, options })) {
@@ -217,10 +224,25 @@ export function sdkAgentDriver(config: SdkDriverConfig = {}): AgentDriver {
finalReport: result.result ?? "",
stoppedByTurnCap: result.subtype === "error_max_turns",
};
} finally {
rmSync(workdir, { recursive: true, force: true });
}
},
};
}
/**
* Create a fresh, empty working directory for one agent run, outside any git
* checkout. The agent's shell runs here so a shell tool that defaults its target
* repository from the local checkout (gitea-axi, tea) cannot silently resolve the
* harness's own repository when the agent omits an explicit `-R`; with no ambient
* checkout the agent must target the repository named in its prompt. The caller
* deletes it when the run ends.
*/
export function createAgentWorkdir(): string {
return mkdtempSync(join(tmpdir(), "bench-agent-cwd-"));
}
/** Assemble the SDK query options for an arm's tool configuration. */
function buildOptions(
arm: ArmDefinition,
@@ -228,6 +250,7 @@ function buildOptions(
turnCap: number,
controller: AbortController,
canUseTool: SdkQueryOptions["canUseTool"],
cwd: string,
): SdkQueryOptions {
const options: SdkQueryOptions = {
model,
@@ -241,6 +264,9 @@ function buildOptions(
// Start from a clean slate: no user/project settings leak tools or config
// into the measured run.
settingSources: [],
// Run outside any checkout so a forgotten -R cannot resolve the harness's own
// repository instead of the seeded throwaway (see createAgentWorkdir).
cwd,
};
if (arm.shell !== null) {
// Lead the agent's PATH with the arm's curated bin directory so only its one