feat: benchmark credential parity, transcript, honest results + read-tier accuracy (tasks 0032, 0033) #36
@@ -112,6 +112,32 @@ describe("buildArm", () => {
|
||||
expect(envValues).toContain("s3cr3t-token");
|
||||
});
|
||||
|
||||
// Behavior: the gitea-axi arm's shell is handed a credential environment
|
||||
// carrying the host and token from the shared access, so its tool is
|
||||
// pre-authenticated without the agent having to discover credentials
|
||||
// (benchmark-harness spec, "Scaffolding"). The access here is built from
|
||||
// independent literals; shell.env must deep-equal exactly the two facts echoed
|
||||
// back under their env-var names (host→GITEA_AXI_API_URL, token→GITEA_AXI_TOKEN),
|
||||
// and nothing else. The keys and mapping are fixed by gitea-axi's own env
|
||||
// contract, not recomputed from arm.ts.
|
||||
it("gives the gitea-axi arm's shell a credential env with the shared host URL and token", () => {
|
||||
const preAuthed: SharedContext = {
|
||||
coords: { owner: "acme", repo: "bench-xyz" },
|
||||
access: { apiUrl: "https://git.example.test", token: "tok-abc123" },
|
||||
};
|
||||
|
||||
const definition = buildArm("gitea-axi", preAuthed, { binRoot, locate });
|
||||
|
||||
const shell = definition.shell;
|
||||
expect(shell).not.toBeNull();
|
||||
if (shell === null) return;
|
||||
|
||||
expect(shell.env).toEqual({
|
||||
GITEA_AXI_API_URL: "https://git.example.test",
|
||||
GITEA_AXI_TOKEN: "tok-abc123",
|
||||
});
|
||||
});
|
||||
|
||||
// Behavior: each non-MCP arm's tool/PATH configuration comes from the guard and
|
||||
// exposes only that arm's allowed binary (benchmark-harness spec, "Tool
|
||||
// isolation" / ADR 0016). The (arm, binary) pairs are independent literals —
|
||||
|
||||
36
bench/arm.ts
36
bench/arm.ts
@@ -40,6 +40,17 @@ export interface ArmShell {
|
||||
path: string;
|
||||
/** The authoritative tool-isolation guard, bound to this arm. */
|
||||
guard: (command: string) => GuardDecision;
|
||||
/**
|
||||
* Credential environment the arm's tool is pre-configured with, merged into
|
||||
* the agent's shell environment on top of {@link path}. This keeps the arms
|
||||
* symmetric on authentication: every arm is handed its host and token the way
|
||||
* its product is really configured, so none pays a turn tax rediscovering how
|
||||
* to authenticate. The gitea-mcp arm gets the equivalent through its MCP
|
||||
* server's env; raw-api uses the token stated in its prompt directly; the
|
||||
* gitea-axi arm is configured through its own env interface here. Empty for an
|
||||
* arm that needs no ambient credentials.
|
||||
*/
|
||||
env: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -172,7 +183,7 @@ function mcpAttachment(context: SharedContext): ArmMcp {
|
||||
* gitea-mcp arm has no shell binary (`provisionArmBin` exposes nothing for it),
|
||||
* so this returns null there and the arm reaches Gitea through its MCP tools.
|
||||
*/
|
||||
function buildShell(arm: Arm, options: BuildArmOptions): ArmShell | null {
|
||||
function buildShell(arm: Arm, context: SharedContext, options: BuildArmOptions): ArmShell | null {
|
||||
if (arm === "gitea-mcp") {
|
||||
return null;
|
||||
}
|
||||
@@ -183,16 +194,37 @@ function buildShell(arm: Arm, options: BuildArmOptions): ArmShell | null {
|
||||
binDir,
|
||||
path: ambient === "" ? binDir : `${binDir}${delimiter}${ambient}`,
|
||||
guard: (command) => guardCommand(arm, command),
|
||||
env: shellEnv(arm, context),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The credential environment a shell arm's tool is pre-configured with. The
|
||||
* gitea-axi arm is handed its host and token through its own env interface
|
||||
* (`GITEA_AXI_API_URL` / `GITEA_AXI_TOKEN`), the symmetric counterpart to the
|
||||
* gitea-mcp arm's server env: both name the same host and token, and both leave
|
||||
* the agent to name the repository per call (gitea-axi via `-R`, gitea-mcp via
|
||||
* each tool's arguments). The tea and raw-api arms need no ambient credentials —
|
||||
* raw-api uses the token stated in its prompt directly in each request, and tea
|
||||
* resolves its own login store — so their env is empty.
|
||||
*/
|
||||
function shellEnv(arm: Arm, context: SharedContext): Record<string, string> {
|
||||
if (arm === "gitea-axi") {
|
||||
return {
|
||||
GITEA_AXI_API_URL: context.access.apiUrl,
|
||||
GITEA_AXI_TOKEN: context.access.token,
|
||||
};
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/** Assemble the single arm definition the runner consumes for the given arm. */
|
||||
export function buildArm(arm: Arm, context: SharedContext, options: BuildArmOptions): ArmDefinition {
|
||||
const systemPrompt = `${basePrompt(context)}\n\n${armBootstrap(arm, context, options)}`;
|
||||
return {
|
||||
arm,
|
||||
systemPrompt,
|
||||
shell: buildShell(arm, options),
|
||||
shell: buildShell(arm, context, options),
|
||||
mcp: arm === "gitea-mcp" ? mcpAttachment(context) : null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -271,7 +271,10 @@ function buildOptions(
|
||||
if (arm.shell !== null) {
|
||||
// Lead the agent's PATH with the arm's curated bin directory so only its one
|
||||
// allowed binary resolves by name; the guard on canUseTool is the authority.
|
||||
options.env = { ...process.env, PATH: arm.shell.path };
|
||||
// Layer the arm's credential env underneath so its tool is pre-authenticated
|
||||
// the way its product is really configured, symmetric to the gitea-mcp
|
||||
// server's env (see ArmShell.env); PATH stays last so it is never overridden.
|
||||
options.env = { ...process.env, ...arm.shell.env, PATH: arm.shell.path };
|
||||
}
|
||||
if (arm.mcp !== null) {
|
||||
options.mcpServers = { [arm.arm]: { type: "stdio", ...arm.mcp.server } };
|
||||
|
||||
@@ -16,7 +16,17 @@ Reach for `gitea-axi` whenever a task touches a Gitea repository's issues, pull
|
||||
- **Over raw Gitea API calls:** it handles auth, pagination, name-to-ID resolution, and review-decision aggregation for you, so you do not hand-roll HTTP.
|
||||
- **Over improvised `git`:** for anything about issues or pull requests as entities (state, reviews, labels, comments) rather than local commits and branches.
|
||||
|
||||
Run it inside a Gitea checkout, or pass `-R OWNER/NAME` (and `--login <name>`) to target a repository explicitly.
|
||||
## Targeting and authentication
|
||||
|
||||
Every command resolves two things: which repository to act on, and which credentials to authenticate with.
|
||||
Get both right on the first call — they are the usual reason a command fails and has to be retried.
|
||||
|
||||
- **Repository.** Inside a Gitea checkout it is taken from the `origin` remote automatically.
|
||||
Outside a checkout you must name it: pass `-R OWNER/NAME` on every command (or set `GITEA_AXI_REPO=OWNER/NAME` once for the session).
|
||||
- **Credentials.** When the environment is pre-configured — `GITEA_AXI_TOKEN` together with `GITEA_AXI_API_URL` — authentication is automatic and you need nothing more.
|
||||
Otherwise credentials come from a `tea` login: pass `--login <name>` (or set `GITEA_AXI_LOGIN=<name>`) unless the checkout's remote already selects one.
|
||||
|
||||
So outside a checkout with the token in the environment, `gitea-axi <command> -R OWNER/NAME …` is all you need; do not go hunting for a config file or a login profile.
|
||||
|
||||
## Command groups
|
||||
|
||||
|
||||
Reference in New Issue
Block a user