fix: pre-authenticate the gitea-axi bench arm via its env interface
The gitea-axi arm was the only shell arm handed no credentials: the runner set only PATH, so the agent had to reverse-engineer the tea-login system — guessing a profile name and hunting for a config file — before any real work, burning ~4 turns per task. Since turns drive cache-read, the benchmark's dominant cost metric, this scaffolding gap alone inflated gitea-axi's cost-equivalent tokens above every other arm. Hand the arm its host and token through gitea-axi's own env interface (GITEA_AXI_API_URL / GITEA_AXI_TOKEN), the symmetric counterpart to the gitea-mcp server's GITEA_HOST / GITEA_ACCESS_TOKEN env: both name the same two facts, and both still leave the agent to name the repository per call. A shell arm now carries a credential env (empty for tea and raw-api, which need none), merged under PATH in the driver. Also strengthen SKILL.md so a cold agent targets and authenticates on the first call: an explicit "Targeting and authentication" section replaces the buried, optional-looking one-liner, spelling out that outside a checkout `-R OWNER/NAME` plus the environment's token is all that is needed — do not go hunting for a config file or login profile. Verified live: create-memory-leak-issue dropped from 10 turns to 3 and its cache-read fell ~3.8x, with the auth flailing gone from the transcript.
This commit is contained in:
36
bench/arm.ts
36
bench/arm.ts
@@ -40,6 +40,17 @@ export interface ArmShell {
|
||||
path: string;
|
||||
/** The authoritative tool-isolation guard, bound to this arm. */
|
||||
guard: (command: string) => GuardDecision;
|
||||
/**
|
||||
* Credential environment the arm's tool is pre-configured with, merged into
|
||||
* the agent's shell environment on top of {@link path}. This keeps the arms
|
||||
* symmetric on authentication: every arm is handed its host and token the way
|
||||
* its product is really configured, so none pays a turn tax rediscovering how
|
||||
* to authenticate. The gitea-mcp arm gets the equivalent through its MCP
|
||||
* server's env; raw-api uses the token stated in its prompt directly; the
|
||||
* gitea-axi arm is configured through its own env interface here. Empty for an
|
||||
* arm that needs no ambient credentials.
|
||||
*/
|
||||
env: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -172,7 +183,7 @@ function mcpAttachment(context: SharedContext): ArmMcp {
|
||||
* gitea-mcp arm has no shell binary (`provisionArmBin` exposes nothing for it),
|
||||
* so this returns null there and the arm reaches Gitea through its MCP tools.
|
||||
*/
|
||||
function buildShell(arm: Arm, options: BuildArmOptions): ArmShell | null {
|
||||
function buildShell(arm: Arm, context: SharedContext, options: BuildArmOptions): ArmShell | null {
|
||||
if (arm === "gitea-mcp") {
|
||||
return null;
|
||||
}
|
||||
@@ -183,16 +194,37 @@ function buildShell(arm: Arm, options: BuildArmOptions): ArmShell | null {
|
||||
binDir,
|
||||
path: ambient === "" ? binDir : `${binDir}${delimiter}${ambient}`,
|
||||
guard: (command) => guardCommand(arm, command),
|
||||
env: shellEnv(arm, context),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The credential environment a shell arm's tool is pre-configured with. The
|
||||
* gitea-axi arm is handed its host and token through its own env interface
|
||||
* (`GITEA_AXI_API_URL` / `GITEA_AXI_TOKEN`), the symmetric counterpart to the
|
||||
* gitea-mcp arm's server env: both name the same host and token, and both leave
|
||||
* the agent to name the repository per call (gitea-axi via `-R`, gitea-mcp via
|
||||
* each tool's arguments). The tea and raw-api arms need no ambient credentials —
|
||||
* raw-api uses the token stated in its prompt directly in each request, and tea
|
||||
* resolves its own login store — so their env is empty.
|
||||
*/
|
||||
function shellEnv(arm: Arm, context: SharedContext): Record<string, string> {
|
||||
if (arm === "gitea-axi") {
|
||||
return {
|
||||
GITEA_AXI_API_URL: context.access.apiUrl,
|
||||
GITEA_AXI_TOKEN: context.access.token,
|
||||
};
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/** Assemble the single arm definition the runner consumes for the given arm. */
|
||||
export function buildArm(arm: Arm, context: SharedContext, options: BuildArmOptions): ArmDefinition {
|
||||
const systemPrompt = `${basePrompt(context)}\n\n${armBootstrap(arm, context, options)}`;
|
||||
return {
|
||||
arm,
|
||||
systemPrompt,
|
||||
shell: buildShell(arm, options),
|
||||
shell: buildShell(arm, context, options),
|
||||
mcp: arm === "gitea-mcp" ? mcpAttachment(context) : null,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user