fix: run the bench agent in a neutral working directory
Some checks failed
CI / test (pull_request) Failing after 52s
Some checks failed
CI / test (pull_request) Failing after 52s
The SDK driver ran the agent with no explicit cwd, so its shell inherited the harness's own checkout. When the agent omitted `-R OWNER/NAME`, the gitea-axi (and tea) CLI defaulted the repository from that local checkout — silently resolving the harness repo instead of the seeded throwaway — and returned a plausible but wrong result (e.g. `count: 0 open of 0 total` for a repo with no issues). This contaminated read-tier scoring for the checkout- defaulting arms and was surfaced by the newly persisted read reports. Give each run a fresh, empty working directory outside any checkout, so a forgotten `-R` errors instead of hitting the wrong repository, and delete it when the run ends.
This commit is contained in:
@@ -17,6 +17,9 @@
|
||||
// recorded in the transcript, so the runner's post-run audit sees what actually
|
||||
// executed — a blocked attempt is realistic wasted effort, not a leak.
|
||||
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ArmDefinition } from "./arm.js";
|
||||
import { foreignToolReason, type ToolUse } from "./audit.js";
|
||||
import type { TokenComponents } from "./result.js";
|
||||
@@ -104,6 +107,8 @@ interface SdkQueryOptions {
|
||||
abortController: AbortController;
|
||||
canUseTool: (toolName: string, input: Record<string, unknown>) => Promise<SdkPermissionResult>;
|
||||
settingSources: string[];
|
||||
/** The agent's shell working directory: a fresh empty dir outside any checkout. */
|
||||
cwd: string;
|
||||
env?: Record<string, string | undefined>;
|
||||
mcpServers?: Record<string, SdkStdioServer>;
|
||||
disallowedTools?: string[];
|
||||
@@ -197,30 +202,47 @@ export function sdkAgentDriver(config: SdkDriverConfig = {}): AgentDriver {
|
||||
return { behavior: "allow", updatedInput: toolInput };
|
||||
};
|
||||
|
||||
const options = buildOptions(input.arm, model, input.turnCap, controller, canUseTool);
|
||||
const workdir = createAgentWorkdir();
|
||||
try {
|
||||
const options = buildOptions(input.arm, model, input.turnCap, controller, canUseTool, workdir);
|
||||
|
||||
let result: SdkResultMessage | undefined;
|
||||
for await (const message of query({ prompt: input.intent, options })) {
|
||||
if (message.type === "result") {
|
||||
result = message as SdkResultMessage;
|
||||
let result: SdkResultMessage | undefined;
|
||||
for await (const message of query({ prompt: input.intent, options })) {
|
||||
if (message.type === "result") {
|
||||
result = message as SdkResultMessage;
|
||||
}
|
||||
}
|
||||
if (result === undefined) {
|
||||
throw new Error("the Agent SDK produced no result message");
|
||||
}
|
||||
}
|
||||
if (result === undefined) {
|
||||
throw new Error("the Agent SDK produced no result message");
|
||||
}
|
||||
|
||||
return {
|
||||
tokens: sumTokens(result),
|
||||
turns: result.num_turns ?? 0,
|
||||
imputedCostUsd: result.total_cost_usd ?? 0,
|
||||
transcript,
|
||||
finalReport: result.result ?? "",
|
||||
stoppedByTurnCap: result.subtype === "error_max_turns",
|
||||
};
|
||||
return {
|
||||
tokens: sumTokens(result),
|
||||
turns: result.num_turns ?? 0,
|
||||
imputedCostUsd: result.total_cost_usd ?? 0,
|
||||
transcript,
|
||||
finalReport: result.result ?? "",
|
||||
stoppedByTurnCap: result.subtype === "error_max_turns",
|
||||
};
|
||||
} finally {
|
||||
rmSync(workdir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a fresh, empty working directory for one agent run, outside any git
|
||||
* checkout. The agent's shell runs here so a shell tool that defaults its target
|
||||
* repository from the local checkout (gitea-axi, tea) cannot silently resolve the
|
||||
* harness's own repository when the agent omits an explicit `-R`; with no ambient
|
||||
* checkout the agent must target the repository named in its prompt. The caller
|
||||
* deletes it when the run ends.
|
||||
*/
|
||||
export function createAgentWorkdir(): string {
|
||||
return mkdtempSync(join(tmpdir(), "bench-agent-cwd-"));
|
||||
}
|
||||
|
||||
/** Assemble the SDK query options for an arm's tool configuration. */
|
||||
function buildOptions(
|
||||
arm: ArmDefinition,
|
||||
@@ -228,6 +250,7 @@ function buildOptions(
|
||||
turnCap: number,
|
||||
controller: AbortController,
|
||||
canUseTool: SdkQueryOptions["canUseTool"],
|
||||
cwd: string,
|
||||
): SdkQueryOptions {
|
||||
const options: SdkQueryOptions = {
|
||||
model,
|
||||
@@ -241,6 +264,9 @@ function buildOptions(
|
||||
// Start from a clean slate: no user/project settings leak tools or config
|
||||
// into the measured run.
|
||||
settingSources: [],
|
||||
// Run outside any checkout so a forgotten -R cannot resolve the harness's own
|
||||
// repository instead of the seeded throwaway (see createAgentWorkdir).
|
||||
cwd,
|
||||
};
|
||||
if (arm.shell !== null) {
|
||||
// Lead the agent's PATH with the arm's curated bin directory so only its one
|
||||
|
||||
Reference in New Issue
Block a user