feat: benchmark credential parity, transcript, honest results + read-tier accuracy (tasks 0032, 0033) #36

Merged
alexion merged 10 commits from task-0032-bench-read-report-persistence into main 2026-07-18 10:28:33 -04:00
4 changed files with 75 additions and 4 deletions
Showing only changes of commit ab59e699c1 - Show all commits

View File

@@ -112,6 +112,32 @@ describe("buildArm", () => {
expect(envValues).toContain("s3cr3t-token"); expect(envValues).toContain("s3cr3t-token");
}); });
// Behavior: the gitea-axi arm's shell is handed a credential environment
// carrying the host and token from the shared access, so its tool is
// pre-authenticated without the agent having to discover credentials
// (benchmark-harness spec, "Scaffolding"). The access here is built from
// independent literals; shell.env must deep-equal exactly the two facts echoed
// back under their env-var names (host→GITEA_AXI_API_URL, token→GITEA_AXI_TOKEN),
// and nothing else. The keys and mapping are fixed by gitea-axi's own env
// contract, not recomputed from arm.ts.
it("gives the gitea-axi arm's shell a credential env with the shared host URL and token", () => {
const preAuthed: SharedContext = {
coords: { owner: "acme", repo: "bench-xyz" },
access: { apiUrl: "https://git.example.test", token: "tok-abc123" },
};
const definition = buildArm("gitea-axi", preAuthed, { binRoot, locate });
const shell = definition.shell;
expect(shell).not.toBeNull();
if (shell === null) return;
expect(shell.env).toEqual({
GITEA_AXI_API_URL: "https://git.example.test",
GITEA_AXI_TOKEN: "tok-abc123",
});
});
// Behavior: each non-MCP arm's tool/PATH configuration comes from the guard and // Behavior: each non-MCP arm's tool/PATH configuration comes from the guard and
// exposes only that arm's allowed binary (benchmark-harness spec, "Tool // exposes only that arm's allowed binary (benchmark-harness spec, "Tool
// isolation" / ADR 0016). The (arm, binary) pairs are independent literals — // isolation" / ADR 0016). The (arm, binary) pairs are independent literals —

View File

@@ -40,6 +40,17 @@ export interface ArmShell {
path: string; path: string;
/** The authoritative tool-isolation guard, bound to this arm. */ /** The authoritative tool-isolation guard, bound to this arm. */
guard: (command: string) => GuardDecision; guard: (command: string) => GuardDecision;
/**
* Credential environment the arm's tool is pre-configured with, merged into
* the agent's shell environment on top of {@link path}. This keeps the arms
* symmetric on authentication: every arm is handed its host and token the way
* its product is really configured, so none pays a turn tax rediscovering how
* to authenticate. The gitea-mcp arm gets the equivalent through its MCP
* server's env; raw-api uses the token stated in its prompt directly; the
* gitea-axi arm is configured through its own env interface here. Empty for an
* arm that needs no ambient credentials.
*/
env: Record<string, string>;
} }
/** /**
@@ -172,7 +183,7 @@ function mcpAttachment(context: SharedContext): ArmMcp {
* gitea-mcp arm has no shell binary (`provisionArmBin` exposes nothing for it), * gitea-mcp arm has no shell binary (`provisionArmBin` exposes nothing for it),
* so this returns null there and the arm reaches Gitea through its MCP tools. * so this returns null there and the arm reaches Gitea through its MCP tools.
*/ */
function buildShell(arm: Arm, options: BuildArmOptions): ArmShell | null { function buildShell(arm: Arm, context: SharedContext, options: BuildArmOptions): ArmShell | null {
if (arm === "gitea-mcp") { if (arm === "gitea-mcp") {
return null; return null;
} }
@@ -183,16 +194,37 @@ function buildShell(arm: Arm, options: BuildArmOptions): ArmShell | null {
binDir, binDir,
path: ambient === "" ? binDir : `${binDir}${delimiter}${ambient}`, path: ambient === "" ? binDir : `${binDir}${delimiter}${ambient}`,
guard: (command) => guardCommand(arm, command), guard: (command) => guardCommand(arm, command),
env: shellEnv(arm, context),
}; };
} }
/**
* The credential environment a shell arm's tool is pre-configured with. The
* gitea-axi arm is handed its host and token through its own env interface
* (`GITEA_AXI_API_URL` / `GITEA_AXI_TOKEN`), the symmetric counterpart to the
* gitea-mcp arm's server env: both name the same host and token, and both leave
* the agent to name the repository per call (gitea-axi via `-R`, gitea-mcp via
* each tool's arguments). The tea and raw-api arms need no ambient credentials —
* raw-api uses the token stated in its prompt directly in each request, and tea
* resolves its own login store — so their env is empty.
*/
function shellEnv(arm: Arm, context: SharedContext): Record<string, string> {
if (arm === "gitea-axi") {
return {
GITEA_AXI_API_URL: context.access.apiUrl,
GITEA_AXI_TOKEN: context.access.token,
};
}
return {};
}
/** Assemble the single arm definition the runner consumes for the given arm. */ /** Assemble the single arm definition the runner consumes for the given arm. */
export function buildArm(arm: Arm, context: SharedContext, options: BuildArmOptions): ArmDefinition { export function buildArm(arm: Arm, context: SharedContext, options: BuildArmOptions): ArmDefinition {
const systemPrompt = `${basePrompt(context)}\n\n${armBootstrap(arm, context, options)}`; const systemPrompt = `${basePrompt(context)}\n\n${armBootstrap(arm, context, options)}`;
return { return {
arm, arm,
systemPrompt, systemPrompt,
shell: buildShell(arm, options), shell: buildShell(arm, context, options),
mcp: arm === "gitea-mcp" ? mcpAttachment(context) : null, mcp: arm === "gitea-mcp" ? mcpAttachment(context) : null,
}; };
} }

View File

@@ -271,7 +271,10 @@ function buildOptions(
if (arm.shell !== null) { if (arm.shell !== null) {
// Lead the agent's PATH with the arm's curated bin directory so only its one // Lead the agent's PATH with the arm's curated bin directory so only its one
// allowed binary resolves by name; the guard on canUseTool is the authority. // allowed binary resolves by name; the guard on canUseTool is the authority.
options.env = { ...process.env, PATH: arm.shell.path }; // Layer the arm's credential env underneath so its tool is pre-authenticated
// the way its product is really configured, symmetric to the gitea-mcp
// server's env (see ArmShell.env); PATH stays last so it is never overridden.
options.env = { ...process.env, ...arm.shell.env, PATH: arm.shell.path };
} }
if (arm.mcp !== null) { if (arm.mcp !== null) {
options.mcpServers = { [arm.arm]: { type: "stdio", ...arm.mcp.server } }; options.mcpServers = { [arm.arm]: { type: "stdio", ...arm.mcp.server } };

View File

@@ -16,7 +16,17 @@ Reach for `gitea-axi` whenever a task touches a Gitea repository's issues, pull
- **Over raw Gitea API calls:** it handles auth, pagination, name-to-ID resolution, and review-decision aggregation for you, so you do not hand-roll HTTP. - **Over raw Gitea API calls:** it handles auth, pagination, name-to-ID resolution, and review-decision aggregation for you, so you do not hand-roll HTTP.
- **Over improvised `git`:** for anything about issues or pull requests as entities (state, reviews, labels, comments) rather than local commits and branches. - **Over improvised `git`:** for anything about issues or pull requests as entities (state, reviews, labels, comments) rather than local commits and branches.
Run it inside a Gitea checkout, or pass `-R OWNER/NAME` (and `--login <name>`) to target a repository explicitly. ## Targeting and authentication
Every command resolves two things: which repository to act on, and which credentials to authenticate with.
Get both right on the first call — they are the usual reason a command fails and has to be retried.
- **Repository.** Inside a Gitea checkout it is taken from the `origin` remote automatically.
Outside a checkout you must name it: pass `-R OWNER/NAME` on every command (or set `GITEA_AXI_REPO=OWNER/NAME` once for the session).
- **Credentials.** When the environment is pre-configured — `GITEA_AXI_TOKEN` together with `GITEA_AXI_API_URL` — authentication is automatic and you need nothing more.
Otherwise credentials come from a `tea` login: pass `--login <name>` (or set `GITEA_AXI_LOGIN=<name>`) unless the checkout's remote already selects one.
So outside a checkout with the token in the environment, `gitea-axi <command> -R OWNER/NAME …` is all you need; do not go hunting for a config file or a login profile.
## Command groups ## Command groups