fix: pre-authenticate the gitea-axi bench arm via its env interface

The gitea-axi arm was the only shell arm handed no credentials: the
runner set only PATH, so the agent had to reverse-engineer the tea-login
system — guessing a profile name and hunting for a config file — before
any real work, burning ~4 turns per task. Since turns drive cache-read,
the benchmark's dominant cost metric, this scaffolding gap alone inflated
gitea-axi's cost-equivalent tokens above every other arm.

Hand the arm its host and token through gitea-axi's own env interface
(GITEA_AXI_API_URL / GITEA_AXI_TOKEN), the symmetric counterpart to the
gitea-mcp server's GITEA_HOST / GITEA_ACCESS_TOKEN env: both name the
same two facts, and both still leave the agent to name the repository per
call. A shell arm now carries a credential env (empty for tea and
raw-api, which need none), merged under PATH in the driver.

Also strengthen SKILL.md so a cold agent targets and authenticates on the
first call: an explicit "Targeting and authentication" section replaces
the buried, optional-looking one-liner, spelling out that outside a
checkout `-R OWNER/NAME` plus the environment's token is all that is
needed — do not go hunting for a config file or login profile.

Verified live: create-memory-leak-issue dropped from 10 turns to 3 and
its cache-read fell ~3.8x, with the auth flailing gone from the transcript.
This commit is contained in:
2026-07-17 15:10:48 -04:00
parent 14d494afa2
commit ab59e699c1
4 changed files with 75 additions and 4 deletions

View File

@@ -112,6 +112,32 @@ describe("buildArm", () => {
expect(envValues).toContain("s3cr3t-token");
});
// Behavior: the gitea-axi arm's shell is handed a credential environment
// carrying the host and token from the shared access, so its tool is
// pre-authenticated without the agent having to discover credentials
// (benchmark-harness spec, "Scaffolding"). The access here is built from
// independent literals; shell.env must deep-equal exactly the two facts echoed
// back under their env-var names (host→GITEA_AXI_API_URL, token→GITEA_AXI_TOKEN),
// and nothing else. The keys and mapping are fixed by gitea-axi's own env
// contract, not recomputed from arm.ts.
it("gives the gitea-axi arm's shell a credential env with the shared host URL and token", () => {
const preAuthed: SharedContext = {
coords: { owner: "acme", repo: "bench-xyz" },
access: { apiUrl: "https://git.example.test", token: "tok-abc123" },
};
const definition = buildArm("gitea-axi", preAuthed, { binRoot, locate });
const shell = definition.shell;
expect(shell).not.toBeNull();
if (shell === null) return;
expect(shell.env).toEqual({
GITEA_AXI_API_URL: "https://git.example.test",
GITEA_AXI_TOKEN: "tok-abc123",
});
});
// Behavior: each non-MCP arm's tool/PATH configuration comes from the guard and
// exposes only that arm's allowed binary (benchmark-harness spec, "Tool
// isolation" / ADR 0016). The (arm, binary) pairs are independent literals —

View File

@@ -40,6 +40,17 @@ export interface ArmShell {
path: string;
/** The authoritative tool-isolation guard, bound to this arm. */
guard: (command: string) => GuardDecision;
/**
* Credential environment the arm's tool is pre-configured with, merged into
* the agent's shell environment on top of {@link path}. This keeps the arms
* symmetric on authentication: every arm is handed its host and token the way
* its product is really configured, so none pays a turn tax rediscovering how
* to authenticate. The gitea-mcp arm gets the equivalent through its MCP
* server's env; raw-api uses the token stated in its prompt directly; the
* gitea-axi arm is configured through its own env interface here. Empty for an
* arm that needs no ambient credentials.
*/
env: Record<string, string>;
}
/**
@@ -172,7 +183,7 @@ function mcpAttachment(context: SharedContext): ArmMcp {
* gitea-mcp arm has no shell binary (`provisionArmBin` exposes nothing for it),
* so this returns null there and the arm reaches Gitea through its MCP tools.
*/
function buildShell(arm: Arm, options: BuildArmOptions): ArmShell | null {
function buildShell(arm: Arm, context: SharedContext, options: BuildArmOptions): ArmShell | null {
if (arm === "gitea-mcp") {
return null;
}
@@ -183,16 +194,37 @@ function buildShell(arm: Arm, options: BuildArmOptions): ArmShell | null {
binDir,
path: ambient === "" ? binDir : `${binDir}${delimiter}${ambient}`,
guard: (command) => guardCommand(arm, command),
env: shellEnv(arm, context),
};
}
/**
* The credential environment a shell arm's tool is pre-configured with. The
* gitea-axi arm is handed its host and token through its own env interface
* (`GITEA_AXI_API_URL` / `GITEA_AXI_TOKEN`), the symmetric counterpart to the
* gitea-mcp arm's server env: both name the same host and token, and both leave
* the agent to name the repository per call (gitea-axi via `-R`, gitea-mcp via
* each tool's arguments). The tea and raw-api arms need no ambient credentials —
* raw-api uses the token stated in its prompt directly in each request, and tea
* resolves its own login store — so their env is empty.
*/
function shellEnv(arm: Arm, context: SharedContext): Record<string, string> {
if (arm === "gitea-axi") {
return {
GITEA_AXI_API_URL: context.access.apiUrl,
GITEA_AXI_TOKEN: context.access.token,
};
}
return {};
}
/** Assemble the single arm definition the runner consumes for the given arm. */
export function buildArm(arm: Arm, context: SharedContext, options: BuildArmOptions): ArmDefinition {
const systemPrompt = `${basePrompt(context)}\n\n${armBootstrap(arm, context, options)}`;
return {
arm,
systemPrompt,
shell: buildShell(arm, options),
shell: buildShell(arm, context, options),
mcp: arm === "gitea-mcp" ? mcpAttachment(context) : null,
};
}

View File

@@ -271,7 +271,10 @@ function buildOptions(
if (arm.shell !== null) {
// Lead the agent's PATH with the arm's curated bin directory so only its one
// allowed binary resolves by name; the guard on canUseTool is the authority.
options.env = { ...process.env, PATH: arm.shell.path };
// Layer the arm's credential env underneath so its tool is pre-authenticated
// the way its product is really configured, symmetric to the gitea-mcp
// server's env (see ArmShell.env); PATH stays last so it is never overridden.
options.env = { ...process.env, ...arm.shell.env, PATH: arm.shell.path };
}
if (arm.mcp !== null) {
options.mcpServers = { [arm.arm]: { type: "stdio", ...arm.mcp.server } };