From b0a0ffc1ab0aaad3880f63fb7119188d42e47fd9 Mon Sep 17 00:00:00 2001 From: alexion Date: Sat, 18 Jul 2026 23:25:32 -0400 Subject: [PATCH 1/7] fix(bench): gate seeding on repo+index readiness and match read counts semantically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two benchmark-harness races were being scored as agent failures. A freshly seeded throwaway repo could 404 for an independent reader (the agent, a fresh process) before Gitea made it consistent, and the async issue indexer lagged so `search issues`/`search prs` returned nothing right after seeding — leaving the agent unable to find the target it was asked to act on. seedRepo now blocks until an independent read confirms the repo is reachable, its full seeded issue and pull spread is visible, and a seeded issue and pull are returned by the search index, before releasing the agent. It polls up to 60s and throws a loud harness error on timeout rather than letting a propagation delay become a scored agent failure. Also add an optional `pattern` regex to RequiredFact so a read answer's count is recognised semantically rather than as a fixed phrase: "5 issues are currently open" no longer fails against the literal "5 issues are open", while the alphabetic-only filler run keeps a wrong count beside the right number (e.g. "5 issues ... 3 open") failing. --- bench/checker.test.ts | 58 ++++++++++++++++++++++ bench/checker.ts | 18 +++++-- bench/scoring-spec.ts | 11 +++++ bench/seed.ts | 111 ++++++++++++++++++++++++++++++++++++++++++ bench/task-suite.ts | 5 ++ 5 files changed, 200 insertions(+), 3 deletions(-) diff --git a/bench/checker.test.ts b/bench/checker.test.ts index ddaebe7..4f3fa36 100644 --- a/bench/checker.test.ts +++ b/bench/checker.test.ts @@ -340,6 +340,64 @@ describe("checkReadAnswer", () => { expect(result.pass).toBe(false); }); + + it("passes via pattern when filler words break every contiguous anyOf phrase", () => { + // A count fact is brittle when pinned to fixed phrases: a human padding the + // answer with filler words ("issues are currently") splits any contiguous + // rendering. The optional `pattern` — a case-insensitive regex source — spans + // that filler as a bounded run of alphabetic words between the number and + // "open", so the fact is satisfied even though no `anyOf` phrase appears verbatim. + const facts: RequiredFact[] = [ + { + description: "count of open issues", + anyOf: ["5 open", "5 issues are open"], + pattern: "\\b5(?: [a-z]+){0,4} open\\b", + }, + ]; + + // "issues are currently" is inserted between "5" and "open", so neither + // contiguous anyOf phrase matches — but the pattern's alphabetic filler run does. + const report = "5 issues are currently open (5 of 5 total)."; + + expect(checkReadAnswer(facts, report)).toEqual({ pass: true }); + }); + + it("fails when a digit interrupts the number-to-open run, so a wrong count cannot slip through", () => { + // The same pattern-bearing fact as above. Here the report states a DIFFERENT + // open count (3), merely mentioning the number 5 elsewhere. The pattern's + // filler run is alphabetic only, so the digit "3" between the matched "5" and + // "open" is not spanned, and no anyOf phrase matches either — the fact fails. + const facts: RequiredFact[] = [ + { + description: "count of open issues", + anyOf: ["5 open", "5 issues are open"], + pattern: "\\b5(?: [a-z]+){0,4} open\\b", + }, + ]; + + // "5 issues in total, and 3 open" — the 5 is a total, the open count is 3. + const report = "There are 5 issues in total, and 3 open."; + + const result = checkReadAnswer(facts, report); + + expect(result.pass).toBe(false); + // The unmet fact must remain identifiable by its own description. + if (result.pass === false) { + expect(result.differences.some((d) => d.includes("count of open issues"))).toBe(true); + } + }); + + it("still matches on anyOf alone when a fact carries no pattern", () => { + // A fact without `pattern` behaves exactly as before: only the contiguous + // anyOf renderings are consulted, unaffected by the new pattern support. + const facts: RequiredFact[] = [ + { description: "count of open issues", anyOf: ["7 open issues", "seven open issues"] }, + ]; + + const report = "The board shows 7 open issues right now."; + + expect(checkReadAnswer(facts, report)).toEqual({ pass: true }); + }); }); describe("score", () => { diff --git a/bench/checker.ts b/bench/checker.ts index c51a217..198a0ed 100644 --- a/bench/checker.ts +++ b/bench/checker.ts @@ -241,9 +241,7 @@ function matchByKey( */ export function checkReadAnswer(facts: RequiredFact[], report: string): CheckResult { const haystack = normalizeText(report); - const missing = facts.filter( - (fact) => !fact.anyOf.some((rendering) => haystack.includes(normalizeText(rendering))), - ); + const missing = facts.filter((fact) => !factPresent(fact, haystack)); if (missing.length === 0) { return { pass: true }; } @@ -253,6 +251,20 @@ export function checkReadAnswer(facts: RequiredFact[], report: string): CheckRes }; } +/** + * Whether a required fact is present in the normalized report: any `anyOf` + * rendering as a contiguous substring, or the optional `pattern` regex matching. + * The pattern is compiled case-insensitively over the already-normalized text, so + * it recognises a count padded with filler ("5 issues are currently open") that no + * fixed phrase would. + */ +function factPresent(fact: RequiredFact, haystack: string): boolean { + if (fact.anyOf.some((rendering) => haystack.includes(normalizeText(rendering)))) { + return true; + } + return fact.pattern !== undefined && new RegExp(fact.pattern, "i").test(haystack); +} + /** * Lower-case, drop markdown emphasis/code markers, and collapse runs of * whitespace so incidental phrasing and formatting do not matter — a report that diff --git a/bench/scoring-spec.ts b/bench/scoring-spec.ts index c8d9638..82dda8e 100644 --- a/bench/scoring-spec.ts +++ b/bench/scoring-spec.ts @@ -131,10 +131,21 @@ export interface RepoState { * whitespace and case normalization), so a count or a name can be phrased * variously without resorting to an LLM judge. `description` names the fact in * diagnostics when it is missing. + * + * `anyOf` matches a *contiguous* substring, which is brittle for facts a human + * naturally pads with filler — "5 issues are currently open" does not contain + * the fixed phrase "5 issues are open". For those, supply `pattern`: a regular + * expression (matched against the same normalized report) that satisfies the + * fact when it matches, so the count itself can be recognised rather than one + * exact wording. A fact is present when any `anyOf` rendering *or* `pattern` + * matches; `anyOf` stays the human-readable renderings even when `pattern` + * carries the real matcher. */ export interface RequiredFact { description: string; anyOf: string[]; + /** Optional regex (source, matched case-insensitively against the normalized report). */ + pattern?: string; } /** diff --git a/bench/seed.ts b/bench/seed.ts index 7ad9a3b..531ef00 100644 --- a/bench/seed.ts +++ b/bench/seed.ts @@ -359,12 +359,122 @@ async function ensurePullRequest( await ensureReviews(access, coords, number, pr.reviews); } +/** How long the readiness gate polls before giving up, and how often it re-checks. */ +const READINESS_TIMEOUT_MS = 60_000; +const READINESS_INTERVAL_MS = 750; + +/** Resolve after `ms` milliseconds. */ +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** A search hit's minimal shape: the repository the matched issue/pull belongs to. */ +interface SearchHit { + repository?: { name?: string } | null; +} + +/** + * Whether the issue indexer has caught up: a full-text search for `title` of the + * given kind returns a hit in this repository. The endpoint spans every repo the + * owner can access (mirroring `gitea-axi search`), so a hit only counts when its + * repository matches. A non-2xx or a not-yet-indexed title reads as not-ready. + */ +async function searchIndexed( + access: BenchAccess, + coords: RepoCoords, + type: "issues" | "pulls", + title: string, +): Promise { + const query = new URLSearchParams({ q: title, type, owner: coords.owner, state: "all", limit: "50" }); + const res = await request(access, "GET", `/repos/issues/search?${query.toString()}`); + if (!res.ok) { + return false; + } + const hits = (await res.json()) as SearchHit[]; + return hits.some((hit) => hit.repository?.name === coords.repo); +} + +/** + * The reason a freshly seeded repository is not yet ready for the agent, or `null` + * when it is. Ready means an independent read sees the repository and its full + * seeded issue and pull spread, and the issue indexer returns a seeded issue and + * pull — the two consistency windows (repo visibility and index lag) an agent + * would otherwise race and fail against. Every read is non-throwing, so a + * transient error reads as not-ready and is retried rather than propagated. + */ +async function readinessGap(access: BenchAccess, coords: RepoCoords): Promise { + const repoPath = `/repos/${coords.owner}/${coords.repo}`; + const repoRes = await request(access, "GET", repoPath); + if (!repoRes.ok) { + return `repository read returned ${repoRes.status}`; + } + const issuesRes = await request(access, "GET", `${repoPath}/issues?type=issues&state=all&limit=100`); + if (!issuesRes.ok) { + return `issue list returned ${issuesRes.status}`; + } + const issues = (await issuesRes.json()) as GiteaIssue[]; + if (issues.length < SEED_PLAN.issues.length) { + return `only ${issues.length}/${SEED_PLAN.issues.length} issues visible`; + } + const pullsRes = await request(access, "GET", `${repoPath}/pulls?state=all&limit=100`); + if (!pullsRes.ok) { + return `pull list returned ${pullsRes.status}`; + } + const pulls = (await pullsRes.json()) as GiteaPull[]; + if (pulls.length < SEED_PLAN.pullRequests.length) { + return `only ${pulls.length}/${SEED_PLAN.pullRequests.length} pull requests visible`; + } + const sampleIssue = SEED_PLAN.issues[0]!.title; + if (!(await searchIndexed(access, coords, "issues", sampleIssue))) { + return `issue "${sampleIssue}" not yet indexed for search`; + } + const samplePull = SEED_PLAN.pullRequests[0]!.title; + if (!(await searchIndexed(access, coords, "pulls", samplePull))) { + return `pull request "${samplePull}" not yet indexed for search`; + } + return null; +} + +/** + * Block until a freshly seeded repository is fully consistent for the agent, or + * fail loudly if it never settles within the timeout. Gitea makes a just-created + * repository and its just-written issues and pulls visible to the seed's own + * writes immediately, but an independent reader — the agent, a fresh process + * moments later — can hit a brief 404 window on the repository and a longer lag on + * the async issue indexer. Both are the benchmark's races, not the tool's, so + * closing them here keeps a scored run measuring gitea-axi rather than host + * propagation. A timeout is a harness failure surfaced to the maintainer, never a + * scored agent failure. + */ +export async function waitForSeedReady( + access: BenchAccess, + coords: RepoCoords, + timeoutMs = READINESS_TIMEOUT_MS, + intervalMs = READINESS_INTERVAL_MS, +): Promise { + const deadline = Date.now() + timeoutMs; + let gap = await readinessGap(access, coords); + while (gap !== null) { + if (Date.now() >= deadline) { + throw new Error( + `seeded repository ${coords.owner}/${coords.repo} not ready after ${timeoutMs}ms: ${gap}`, + ); + } + await delay(intervalMs); + gap = await readinessGap(access, coords); + } +} + /** * Seed a freshly provisioned repository to the ground truth, idempotently. Labels * come first (so issues and pull requests can apply them), then the issue spread, * then the pull requests. Returns the deterministic ground-truth RepoState the * checker scores against; on a fresh repository the created numbers match it, * and a re-run leaves them unchanged. + * + * Before returning, it waits out the repository-visibility and issue-indexer + * consistency windows (see waitForSeedReady), so the agent that runs next never + * races a repository that is not yet readable or searchable. */ export async function seedRepo(access: BenchAccess, coords: RepoCoords): Promise { const user = await currentUser(access); @@ -384,6 +494,7 @@ export async function seedRepo(access: BenchAccess, coords: RepoCoords): Promise await ensurePullRequest(access, coords, pr, labelIds, pullsByTitle); } + await waitForSeedReady(access, coords); return groundTruth(user); } diff --git a/bench/task-suite.ts b/bench/task-suite.ts index 832e1f4..bbd25ad 100644 --- a/bench/task-suite.ts +++ b/bench/task-suite.ts @@ -92,6 +92,11 @@ function readTasks(): BenchTask[] { { description: "the repository has 5 open issues", anyOf: ["5 open", "five open", "open issues: 5", "open: 5", "5 issues are open"], + // "5" followed by "open" across up to four alphabetic filler words, so + // natural padding ("5 issues are currently open") is recognised while a + // digit between them (a wrong "5 issues, 3 open") is not: the filler run + // is alphabetic only, so it cannot span another count. + pattern: "\\b5(?: [a-z]+){0,4} open\\b", }, ]), }, -- 2.47.3 From f9125e099db0c642b8a6b41fee092ad516d4890b Mon Sep 17 00:00:00 2001 From: alexion Date: Sat, 18 Jul 2026 23:25:32 -0400 Subject: [PATCH 2/7] docs: correct bench/CLI login name and record bench test-config gotcha MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tea login store here holds `alexion` and `csv-reviewer`, not `axi`; `selectLogin` matches `--login` by exact name, so `--login alexion` works and an unknown `--login axi` fails with VALIDATION_ERROR — the prior note had this backwards. Also record that bench/ unit tests run only under vitest.bench.config.ts (plain `vitest run` matches test/** and finds none). --- CLAUDE.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 69cd75d..e9a3b8f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,10 +13,15 @@ The benchmark arms invoke the **built `dist/main.js`** (the `gitea-axi` binary o Run `npm run build` before any live `bench:run` if you want `src/` changes reflected; the bench does not run from source. Prefer this project's own CLI for pull requests — it is the tool being built, so opening its PRs with it is the dogfood path: -`npm run build && node dist/main.js pr create --login axi --base main --head --title --body-file `. -The login profile is named `axi`, not `alexion` — passing `--login alexion` fails with `VALIDATION_ERROR`. -It reuses the `tea` login profiles, so it needs no separate credentials. -Fall back to `tea pr create --login axi --base main --head ` only for what gitea-axi cannot do yet; `tea pr` still lists PRs until `pr list` lands (task 0008). +`npm run build && node dist/main.js pr create --login alexion --base main --head --title --body-file `. +It reuses the `tea` login store, which here holds exactly `alexion` and `csv-reviewer` — there is no `axi` profile. +`selectLogin` matches the `--login` value against those names exactly, so `--login alexion` works and an unknown name like `--login axi` fails with `VALIDATION_ERROR` ("Login profile "axi" not found"). +Fall back to `tea pr create --login alexion --base main --head ` only for what gitea-axi cannot do yet; `tea pr` still lists PRs until `pr list` lands (task 0008). + +The same login store backs the benchmark: `npm run bench:run -- --arm --login alexion --task ` (or set `GITEA_AXI_BENCH_LOGIN=alexion`). + +The `bench/` unit tests only run under their own Vitest project config: `npx vitest run --config vitest.bench.config.ts bench/.test.ts`. +Plain `npx vitest run bench/.test.ts` reports "no tests" because the default `vitest.config.ts` includes only `test/**`. Task branches are merged into `main` on the remote, so the local `main` goes stale. Always `git fetch origin` and cut a task branch from `origin/main`, not from whatever local `main` happens to point at. -- 2.47.3 From dd58cd9dad7ae7395d65e81b32c7f5628a2e86be Mon Sep 17 00:00:00 2001 From: alexion Date: Sun, 19 Jul 2026 09:41:13 -0400 Subject: [PATCH 3/7] fix(context): tolerate a trailing /api/v1 in the base URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The client (gitea-js) appends /api/v1 to the base URL itself, so a GITEA_AXI_API_URL that already carries it — a natural guess given the variable's name — doubled the segment and failed as a spurious REPO_NOT_FOUND. Normalize the base URL by stripping a trailing /api/v1 (and any trailing slashes) on both the env-URL and tea-login paths, so the host base and the /api/v1 endpoint both resolve. --- src/context.ts | 18 ++++++++++++++--- test/context.test.ts | 48 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/context.ts b/src/context.ts index 1049b78..120d487 100644 --- a/src/context.ts +++ b/src/context.ts @@ -60,6 +60,17 @@ function hostnameOf(url: string, origin: string): string { } } +/** + * Normalize a Gitea base URL to the host root the client expects. The client + * (gitea-js) appends `/api/v1` itself, so a value that already carries it — a + * natural guess when the variable is literally named `..._API_URL` — would double + * the segment and 404 as a spurious `REPO_NOT_FOUND`. Strip a trailing `/api/v1` + * (with any trailing slashes) so the host base and the API endpoint both work. + */ +function normalizeApiBase(url: string): string { + return url.replace(/\/+$/, "").replace(/\/api\/v1$/, ""); +} + function resolveTestModeContext( deps: CliDeps, apiUrl: string, @@ -72,10 +83,11 @@ function resolveTestModeContext( ["Set `GITEA_AXI_REPO=OWNER/NAME` or pass `-R OWNER/NAME`"], ); } + const base = normalizeApiBase(apiUrl); return { ...parseRepoSpec(overrides.repoSpec, overrides.repoOrigin), - host: hostnameOf(apiUrl, "`GITEA_AXI_API_URL`"), - apiUrl: apiUrl.replace(/\/+$/, ""), + host: hostnameOf(base, "`GITEA_AXI_API_URL`"), + apiUrl: base, token: deps.env.GITEA_AXI_TOKEN ?? "", repoSource: overrides.repoSource, loginSource: overrides.loginSource, @@ -184,7 +196,7 @@ export async function resolveRepoContext(deps: CliDeps): Promise { owner, name, host, - apiUrl: login.url.replace(/\/+$/, ""), + apiUrl: normalizeApiBase(login.url), token, repoSource: overrides.repoSource, loginSource: overrides.loginSource, diff --git a/test/context.test.ts b/test/context.test.ts index 1e2ec3f..55282c0 100644 --- a/test/context.test.ts +++ b/test/context.test.ts @@ -1,4 +1,6 @@ import { afterEach, describe, expect, it } from "vitest"; +import { resolveRepoContext } from "../src/context.js"; +import type { CliDeps } from "../src/deps.js"; import { startFixtureServer, type FixtureServer } from "./fixture-server.js"; import { runCliTest, testModeEnv } from "./harness.js"; @@ -117,3 +119,49 @@ describe("context overrides", () => { expect(exitCode).toBe(0); }); }); + +describe("apiUrl normalization", () => { + function depsWithApiUrl(apiUrl: string): CliDeps { + return { + env: { + GITEA_AXI_API_URL: apiUrl, + GITEA_AXI_REPO: "acme/widgets", + GITEA_AXI_TOKEN: "test-token", + }, + cwd: process.cwd(), + globals: {}, + }; + } + + it("strips a trailing /api/v1 suffix from the host base", async () => { + const context = await resolveRepoContext( + depsWithApiUrl("https://git.example.com/api/v1"), + ); + + expect(context.apiUrl).toBe("https://git.example.com"); + }); + + it("strips a trailing /api/v1/ with a trailing slash", async () => { + const context = await resolveRepoContext( + depsWithApiUrl("https://git.example.com/api/v1/"), + ); + + expect(context.apiUrl).toBe("https://git.example.com"); + }); + + it("leaves a host base without an /api/v1 suffix unchanged", async () => { + const context = await resolveRepoContext( + depsWithApiUrl("https://git.example.com"), + ); + + expect(context.apiUrl).toBe("https://git.example.com"); + }); + + it("strips a lone trailing slash from the host base", async () => { + const context = await resolveRepoContext( + depsWithApiUrl("https://git.example.com/"), + ); + + expect(context.apiUrl).toBe("https://git.example.com"); + }); +}); -- 2.47.3 From d56b1d0707251e9a1eaaadb5b4660a4f1bec8460 Mon Sep 17 00:00:00 2001 From: alexion Date: Sun, 19 Jul 2026 09:41:13 -0400 Subject: [PATCH 4/7] refactor(skill): steer find-then-act instead of open-ended discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bundled skill's Discovery section told the agent to run the bare dashboard and reach for --help proactively, and advertised overlapping find-paths — inducing exploratory commands that made the gitea-axi arm the most expensive of the benchmark's four. Replace it with a "find the target, then act" section, name the non-obvious mutation flags so common edits do not need --help, and drop the setup line and the over-tea/raw/git bullets that only duplicated the description. A same-time A/B cut cost-equivalent tokens ~10% and collapsed bare-dashboard use from 60% to 7% with no loss of success. --- skills/gitea-axi/SKILL.md | 51 ++++++++++++++++----------------------- test/skill.test.ts | 18 ++++++++------ 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/skills/gitea-axi/SKILL.md b/skills/gitea-axi/SKILL.md index 5e30a48..02951b3 100644 --- a/skills/gitea-axi/SKILL.md +++ b/skills/gitea-axi/SKILL.md @@ -6,43 +6,34 @@ description: Use when working with a Gitea repository's issues, pull requests, l # gitea-axi `gitea-axi` is an agent-ergonomic CLI for a Gitea repository's issues and pull requests. -Its output is compact TOON built for another program to read, and its errors are structured with actionable suggestions. - -## When to use it - -Reach for `gitea-axi` whenever a task touches a Gitea repository's issues, pull requests, labels, or reviews. - -- **Over `tea`:** `gitea-axi` returns structured output and typed errors instead of human-formatted tables, and it defaults the repository and login from the local checkout. -- **Over raw Gitea API calls:** it handles auth, pagination, name-to-ID resolution, and review-decision aggregation for you, so you do not hand-roll HTTP. -- **Over improvised `git`:** for anything about issues or pull requests as entities (state, reviews, labels, comments) rather than local commits and branches. +Its output is compact TOON meant to be read directly, and a failed command's error names the fix — follow that suggestion rather than guessing at another command. ## Targeting and authentication -Every command resolves two things: which repository to act on, and which credentials to authenticate with. -Get both right on the first call — they are the usual reason a command fails and has to be retried. +Every command resolves a repository and credentials; getting both right on the first call is the difference between one command and a retry. -- **Repository.** Inside a Gitea checkout it is taken from the `origin` remote automatically. - Outside a checkout you must name it: pass `-R OWNER/NAME` on every command (or set `GITEA_AXI_REPO=OWNER/NAME` once for the session). -- **Credentials.** When the environment is pre-configured — `GITEA_AXI_TOKEN` together with `GITEA_AXI_API_URL` — authentication is automatic and you need nothing more. - Otherwise credentials come from a `tea` login: pass `--login ` (or set `GITEA_AXI_LOGIN=`) unless the checkout's remote already selects one. +- **Repository.** Inside a Gitea checkout it comes from the `origin` remote. + Outside one, pass `-R OWNER/NAME` on every command, or set `GITEA_AXI_REPO=OWNER/NAME` once for the session. +- **Credentials.** With `GITEA_AXI_TOKEN` and `GITEA_AXI_API_URL` set, authentication is automatic. + Otherwise pass `--login `, or set `GITEA_AXI_LOGIN`. -So outside a checkout with the token in the environment, `gitea-axi -R OWNER/NAME …` is all you need; do not go hunting for a config file or a login profile. +Outside a checkout with the token in the environment, `gitea-axi -R OWNER/NAME …` is the whole invocation — don't look for a config file or a login profile. -## Command groups +## Commands -- `issue` — list, view, create, comment on, edit, close/reopen, pin, and link issues. -- `pr` — create, view, comment on, edit, review, merge, check out, diff, and inspect the checks of pull requests. -- `label` — list, create, edit, and delete labels. -- `search` — full-text search; it takes a subcommand, so search issues with `search issues ""` and pull requests with `search prs ""` (a bare `search ""` is not valid). -- `setup` — install this skill (`setup`) and, opt-in, the SessionStart dashboard hook (`setup hooks`). +- `issue` — list, view, create, comment, edit, close, reopen, pin, and link (blocks / blocked-by). +- `pr` — list, view, create, comment, edit, review, merge, close, reopen, diff, checks, and checkout. +- `label` — list, create, edit, delete. +- `search issues ""` and `search prs ""` — full-text search (a bare `search ""` is not valid). -To read one issue's fields, reach straight for `issue view `: it shows labels and state by default, and takes `--fields assignees,milestone,…` for the rest. -You rarely need `issue list` to answer a question about a single issue. +## Finding and acting -## Discovery +Find the target, then act on it — two commands, not a survey of the repository. -This skill is a pointer, not a command reference — the CLI is the single source of truth for its own interface. - -- Run `gitea-axi` with no arguments for the repository dashboard (open issues and pull requests). - Add `--full` for the open-PR table and issue counts by label. -- Run `gitea-axi --help` (or `gitea-axi --help`) for the exact flags of any command. +- **Find it.** If you already know the number, act on it directly. + Otherwise reach for one command — `search issues ""` for a title or keyword, or `issue list --state all --label ` to narrow by a property — not both. +- **Read one issue or PR.** `issue view ` (or `pr view `) shows labels and state by default, and takes `--fields assignees,milestone,…` for the rest. + You do not need `issue list` to answer a question about a single known issue. +- **Act on it.** `issue edit ` and `pr edit ` change fields with repeatable `--add-label` / `--remove-label` and `--add-assignee` / `--remove-assignee`, plus `--title`, `--body`, and `--milestone`. + Reviewing is `pr review ` with exactly one of `--approve`, `--request-changes`, or `--comment`, and an optional `--body`. + A comment is `issue comment --body `; a new label is `label create --name --color `. diff --git a/test/skill.test.ts b/test/skill.test.ts index bde0c3a..b8942db 100644 --- a/test/skill.test.ts +++ b/test/skill.test.ts @@ -36,19 +36,23 @@ describe("bundled Agent Skill markdown", () => { it("references each command group as a one-liner", () => { const body = skill.toLowerCase(); - for (const group of ["issue", "pr", "label", "search", "setup"]) { + for (const group of ["issue", "pr", "label", "search"]) { expect(body, `expected the skill to mention the ${group} command group`).toContain( group, ); } }); - it("points at the bare dashboard and per-command help for discovery", () => { + it("steers find-then-act and does not push exploratory discovery", () => { const body = skill.toLowerCase(); - // Bare dashboard: running the binary with no arguments. - expect(body).toContain("no argument"); - expect(body).toContain("dashboard"); - // Per-command help. - expect(body).toContain("--help"); + // The intended steering: find the target, then act on it. + expect(body).toContain("find the target"); + expect(body).toContain("act on it"); + // The find/act discipline: not a full survey, and not both find commands at once. + expect(body).toContain("not a survey"); + expect(body).toContain("not both"); + // The removed exploratory anti-pattern must be gone. + expect(body).not.toContain("dashboard"); + expect(body).not.toContain("no argument"); }); }); -- 2.47.3 From 0b0186674e8e36736d1f5b78ab8d1d0b714472d9 Mon Sep 17 00:00:00 2001 From: alexion Date: Sun, 19 Jul 2026 09:46:27 -0400 Subject: [PATCH 5/7] feat(bench): add --skill to override the gitea-axi arm's bundled skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bench:run gains a --skill flag, threaded into the existing BuildArmOptions.skillPath, so a skill variant can be A/B'd against the shipped SKILL.md with the same binary and harness — the mechanism used to validate the find-then-act rewrite — without mutating the shipped file. --- bench/run.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bench/run.ts b/bench/run.ts index e92913c..990bda7 100644 --- a/bench/run.ts +++ b/bench/run.ts @@ -57,6 +57,8 @@ export interface RunArgs { turnCap: number; wallClockMs: number; storeRoot: string; + /** Optional bundled-skill override for the gitea-axi arm; defaults to the shipped SKILL.md. */ + skillPath?: string; } /** The parse outcome: a request for help, or a resolved configuration to run. */ @@ -71,6 +73,7 @@ const KNOWN_FLAGS = new Set([ "turn-cap", "wall-clock-ms", "store", + "skill", ]); /** A usage error, surfaced to the maintainer with the offending detail. */ @@ -157,6 +160,7 @@ export function parseRunArgs( turnCap, wallClockMs, storeRoot: flags.get("store") ?? DEFAULT_STORE_ROOT, + ...(flags.has("skill") ? { skillPath: flags.get("skill") } : {}), }; } @@ -180,6 +184,7 @@ Options: --turn-cap Per-run turn cap (default: ${DEFAULT_TURN_CAP}) --wall-clock-ms Per-run wall-clock backstop in ms (default: ${DEFAULT_WALL_CLOCK_MS}) --store Sample store root (default: ${DEFAULT_STORE_ROOT}) + --skill Override the gitea-axi arm's bundled skill (default: shipped SKILL.md) -h, --help Show this help`; /** Render the run-loop tally into the lines printed after a sitting. */ @@ -246,7 +251,7 @@ export async function runBenchCommand( driver: sdkAgentDriver(), store, bounds: { turnCap: parsed.turnCap, wallClockMs: parsed.wallClockMs }, - build: { binRoot }, + build: { binRoot, ...(parsed.skillPath !== undefined ? { skillPath: parsed.skillPath } : {}) }, }); for (const line of summarize(result, parsed.storeRoot)) { out(line); -- 2.47.3 From 5422f49f388d7be711d1d9d46f5b153bd31f2eef Mon Sep 17 00:00:00 2001 From: alexion Date: Sun, 19 Jul 2026 10:17:14 -0400 Subject: [PATCH 6/7] feat(search): guide a 0-result search and fill the single-match number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The next-step suggestion on `search issues`/`search prs` is now conditioned on the in-repo match count. On a miss it pointed the agent at `view `, which is nonsensical when nothing matched; it now suggests the non-indexed `issue list --state all` / `pr list --state all` fallback, which recovers from both an over-narrow query and issue-indexer lag without naming the cause. On exactly one match it fills the real number (`issue view 2`), applying AXI Principle 9's single-id fill. Two or more matches keep the parameterized placeholder. Search stays a locator — it never auto-loads the detail even on a single match; ADR 0017 records that decision (a deliberate narrowing of Principle 4) and the CONTEXT.md search term is updated to match. --- .claude/CONTEXT.md | 2 + .claude/adr/0017-search-stays-a-locator.md | 26 ++++ src/commands/search.ts | 23 +++- test/search.test.ts | 146 +++++++++++++++++++++ 4 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 .claude/adr/0017-search-stays-a-locator.md diff --git a/.claude/CONTEXT.md b/.claude/CONTEXT.md index c2a00ef..f2fefdb 100644 --- a/.claude/CONTEXT.md +++ b/.claude/CONTEXT.md @@ -99,6 +99,8 @@ _Avoid_: depends, depends-on, dependencies **search**: The full-text query commands (`search issues `, `search prs `), repo-scoped via `owner` param plus [[client-side filtering]] by repository (Gitea's `/repos/issues/search` has no repo-name filter). Results use a locator schema (`number`, `title`, `state`, `author`, `created`) — search finds the number; `issue view` / `pr view` load the detail. +The [[next-step suggestion]] is conditioned on the in-repo match count: zero matches point at the non-indexed `issue list --state all` / `pr list --state all` fallback ("to list all … instead"), which recovers from both an over-narrow query and issue-indexer lag; exactly one match fills the real number (`issue view `, Principle 9's single-id fill); two or more keep the parameterized `` placeholder. +Search never auto-loads the detail even on a single match — it stays a locator (see ADR 0017). The forbidden `--search` flag on the list commands redirects here. _Avoid_: query command, find diff --git a/.claude/adr/0017-search-stays-a-locator.md b/.claude/adr/0017-search-stays-a-locator.md new file mode 100644 index 0000000..6d11240 --- /dev/null +++ b/.claude/adr/0017-search-stays-a-locator.md @@ -0,0 +1,26 @@ +# Search stays a locator; a single match does not auto-load its detail + +`search issues` / `search prs` always return a locator list (`number`, `title`, `state`, `author`, `created`) plus a next-step suggestion — never the full detail, even when exactly one result matches. +This deliberately narrows a literal reading of AXI Principle 4 ("eliminate round trips"). + +## Considered Options + +**Auto-collapse to `view` on a single match** (rejected) — On exactly one result, run `issue view` / `pr view` and return the detail record, sparing the agent a second command. +It reads as the purest Principle 4 outcome, and it is what prompted this decision. +But the agent that searches most often wants the *number* to feed a mutation (`edit`, `close`, `comment`), not the body — so auto-loading the detail spends exactly the body tokens Principle 3's truncation exists to avoid, taxing the common find-then-act path to save a step on the rarer find-then-read one. +It also makes the output shape non-uniform — a list for zero and 2+ matches, a detail record for one — which the agent can no longer rely on. + +**Stay a locator, suggest the next step** (chosen) — Search's job is finding the number to feed into `view` / `edit` (the spec's locator-schema rationale). +On a single match the next-step suggestion fills the real number (`issue view 2`), applying Principle 9's single-id fill; the agent decides whether that number feeds a `view`, an `edit`, or a `close`. + +## The dividing line + +Principle 4 eliminates a *redundant* round trip — a mutation returns the entity it just wrote, so no follow-up `view` is needed (ADR 0008). +`search` → `view` is not redundant: the follow-up is optional and its intent (read vs. act) is the agent's to choose, so collapsing it means guessing intent and over-fetching when the guess is wrong. + +## Consequences + +- `search` output shape is uniform across all match counts: always a locator list with a `help[N]:` next step. +- The next-step suggestion is conditioned on the in-repo match count: 0 → `list --state all` fallback ("to list all … instead"); 1 → `view ` with the real number; 2+ → `view ` placeholder. +- The zero-match fallback points at the non-indexed list, so it recovers from both an over-narrow query and issue-indexer lag without the command having to tell the two apart. +- An agent that does want the detail spends one more command (`view `) by design — the number is already in hand, and it pays only for the detail it actually asks for. diff --git a/src/commands/search.ts b/src/commands/search.ts index a7cf301..150f460 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -107,6 +107,10 @@ interface SearchKind { noun: string; /** The command a matched number feeds into. */ viewCommand: string; + /** The list command suggested as the fallback when a search finds nothing. */ + listCommand: string; + /** Human plural for the fallback note, e.g. "issues" or "pull requests". */ + things: string; /** The `--help` text for this variant. */ help: string; } @@ -116,6 +120,8 @@ const SEARCH_ISSUES: SearchKind = { type: "issues", noun: "issues", viewCommand: "issue view", + listCommand: "issue list", + things: "issues", help: SEARCH_ISSUES_HELP, }; @@ -124,6 +130,8 @@ const SEARCH_PRS: SearchKind = { type: "pulls", noun: "pull_requests", viewCommand: "pr view", + listCommand: "pr list", + things: "pull requests", help: SEARCH_PRS_HELP, }; @@ -217,11 +225,24 @@ async function runSearch(deps: CliDeps, args: string[], kind: SearchKind): Promi extractRow(issue, [...SEARCH_FIELDS, ...extraFields], { now, host: context.host, full }), ); + // The next-step suggestion is conditioned on the match count. On a miss the + // `view` hint is nonsensical, so point at the non-indexed list as a fallback + // (it recovers from both an over-narrow query and index lag); on a single match + // fill the real number (Principle 9's single-id fill); otherwise leave the + // number parameterized. Search stays a locator either way — it never auto-loads + // the detail (see ADR 0017). + const suggestion = + total === 0 + ? suggestCommand(context, `${kind.listCommand} --state all`, `to list all ${kind.things} instead`) + : total === 1 + ? suggestCommand(context, `${kind.viewCommand} ${matches[0]!.number}`, "to see it in full") + : suggestCommand(context, `${kind.viewCommand} `, "to see a match in full"); + return renderList({ noun: kind.noun, rows, countLine: formatCountLine(rows.length, total, false), - help: [suggestCommand(context, `${kind.viewCommand} `, "to see a match in full")], + help: [suggestion], }); } diff --git a/test/search.test.ts b/test/search.test.ts index 7143900..ad1c77b 100644 --- a/test/search.test.ts +++ b/test/search.test.ts @@ -402,3 +402,149 @@ describe("search empty results", () => { }); } }); + +/** + * The `help[1]:` next-step line is emitted by `suggestCommand`, so it appears + * wrapped in a `Run \`gitea-axi …\`` line with `-R`/`--login` normalization. + * These tests pull that help block out of the output and assert on the + * count-conditional suggestion within it. + */ +function helpBlock(stdout: string): string { + const lines = stdout.split("\n"); + const start = lines.findIndex((line) => /^help\[\d+\]:/.test(line)); + expect(start).toBeGreaterThanOrEqual(0); + // The block runs from the help[N]: header to the next blank line / EOF. + const rest = lines.slice(start); + const end = rest.findIndex((line, i) => i > 0 && line.trim() === ""); + return (end === -1 ? rest : rest.slice(0, end)).join("\n"); +} + +describe("search issues count-conditional next-step suggestion", () => { + it("suggests the list fallback when there are zero in-repo matches", async () => { + server = await startFixtureServer([ + { method: "GET", path: SEARCH_PATH, headers: { "X-Total-Count": "0" }, body: [] }, + ]); + + const { stdout, exitCode } = await runCliTest( + ["search", "issues", "login bug"], + { env: testModeEnv(server.url) }, + ); + + expect(exitCode).toBe(0); + const help = helpBlock(stdout); + expect(help).toContain("issue list --state all"); + expect(help).toContain("to list all issues instead"); + expect(help).not.toContain("issue view"); + }); + + it("fills the real number when there is exactly one in-repo match", async () => { + server = await startFixtureServer([ + { + method: "GET", + path: SEARCH_PATH, + headers: { "X-Total-Count": "1" }, + body: [searchIssueOf(2, { title: "Fix login redirect loop" })], + }, + ]); + + const { stdout, exitCode } = await runCliTest( + ["search", "issues", "login bug"], + { env: testModeEnv(server.url) }, + ); + + expect(exitCode).toBe(0); + const help = helpBlock(stdout); + expect(help).toContain("issue view 2"); + expect(help).toContain("to see it in full"); + expect(help).not.toContain("issue view "); + }); + + it("keeps the placeholder when there are two or more in-repo matches", async () => { + server = await startFixtureServer([ + { + method: "GET", + path: SEARCH_PATH, + headers: { "X-Total-Count": "2" }, + body: [ + searchIssueOf(42, { title: "Fix login redirect loop" }), + searchIssueOf(41, { title: "Login button unresponsive" }), + ], + }, + ]); + + const { stdout, exitCode } = await runCliTest( + ["search", "issues", "login bug"], + { env: testModeEnv(server.url) }, + ); + + expect(exitCode).toBe(0); + const help = helpBlock(stdout); + expect(help).toContain("issue view "); + expect(help).toContain("to see a match in full"); + }); +}); + +describe("search prs count-conditional next-step suggestion", () => { + it("suggests the list fallback when there are zero in-repo matches", async () => { + server = await startFixtureServer([ + { method: "GET", path: SEARCH_PATH, headers: { "X-Total-Count": "0" }, body: [] }, + ]); + + const { stdout, exitCode } = await runCliTest( + ["search", "prs", "flaky ci"], + { env: testModeEnv(server.url) }, + ); + + expect(exitCode).toBe(0); + const help = helpBlock(stdout); + expect(help).toContain("pr list --state all"); + expect(help).toContain("to list all pull requests instead"); + expect(help).not.toContain("pr view"); + }); + + it("fills the real number when there is exactly one in-repo match", async () => { + server = await startFixtureServer([ + { + method: "GET", + path: SEARCH_PATH, + headers: { "X-Total-Count": "1" }, + body: [searchIssueOf(2, { title: "Retry flaky CI jobs" })], + }, + ]); + + const { stdout, exitCode } = await runCliTest( + ["search", "prs", "flaky ci"], + { env: testModeEnv(server.url) }, + ); + + expect(exitCode).toBe(0); + const help = helpBlock(stdout); + expect(help).toContain("pr view 2"); + expect(help).toContain("to see it in full"); + expect(help).not.toContain("pr view "); + }); + + it("keeps the placeholder when there are two or more in-repo matches", async () => { + server = await startFixtureServer([ + { + method: "GET", + path: SEARCH_PATH, + headers: { "X-Total-Count": "2" }, + body: [ + searchIssueOf(73, { title: "Retry flaky CI jobs" }), + searchIssueOf(72, { title: "Stabilize CI runners" }), + ], + }, + ]); + + const { stdout, exitCode } = await runCliTest( + ["search", "prs", "flaky ci"], + { env: testModeEnv(server.url) }, + ); + + expect(exitCode).toBe(0); + const help = helpBlock(stdout); + expect(help).toContain("pr view "); + expect(help).toContain("to see a match in full"); + }); +}); -- 2.47.3 From 408956cf328737e7b42546c57913f15c8ccb024a Mon Sep 17 00:00:00 2001 From: alexion Date: Sun, 19 Jul 2026 12:31:05 -0400 Subject: [PATCH 7/7] docs: rewrite bench results for the post-fix 4-arm snapshot Refresh the benchmark README from a clean co-temporal 4-arm run taken after the find-then-act skill rewrite, the /api/v1 tolerance, and the search next-step fix. gitea-axi drops from most-expensive arm to co-leader: within ~1% of raw REST overall, cheapest structured interface by ~23%, cheapest arm outright on the read and find-then-act tiers, at 100% success and the fewest turns of any arm. --- bench/README.md | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/bench/README.md b/bench/README.md index b9f1341..97b0d02 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,8 +1,8 @@ # Benchmark harness This directory holds the benchmark that tests gitea-axi's central claim — that it is an agent-ergonomic, low-token interface to Gitea — against the `tea` CLI, the official `gitea-mcp` server, and raw Gitea REST calls. -The result is honest rather than flattering: gitea-axi is the lowest-cost of the *structured* interfaces — it beats both `tea` and `gitea-mcp` on every tier at 100% task success — but hand-rolled raw REST is cheaper still, because terse HTTP is the token floor no wrapper undercuts. -Keeping the raw-REST arm in the comparison is deliberate: a benchmark of agent-CLIs that omits it will always crown the wrapper, and this one refuses to. +The result is that gitea-axi has reached the token floor: it is the cheapest of the structured interfaces by a wide margin and now runs neck-and-neck with hand-rolled raw REST — within ~1% overall — at 100% task success and the fewest turns of any arm. +Keeping the raw-REST arm in the comparison is deliberate: a benchmark of agent-CLIs that omits the hand-rolled baseline will always flatter the wrapper, and this one refuses to — which is exactly what makes gitea-axi matching that baseline meaningful. ## How it works @@ -15,23 +15,25 @@ Every arm is credentialed the way its product is really configured — the token | arm | cost-equivalent tokens | raw tokens | turns | success | imputed cost | | --- | ---: | ---: | ---: | ---: | ---: | -| raw-api | 16,971 | 52,586 | 4.3 | 100% | ~$0.11 | -| gitea-axi | 19,240 | 81,067 | 6.0 | 100% | ~$0.12 | -| tea | 20,568 | 82,188 | 6.2 | 97% | ~$0.12 | -| gitea-mcp | 21,803 | 79,961 | 5.7 | 100% | ~$0.14 | +| raw-api | 17,613 | 60,384 | 5.0 | 100% | ~$0.11 | +| gitea-axi | 17,815 | 62,705 | 4.8 | 100% | ~$0.11 | +| gitea-mcp | 23,198 | 76,378 | 5.2 | 100% | ~$0.15 | +| tea | 23,210 | 94,600 | 7.2 | 85% | ~$0.14 | -All four arms completed the full matrix — 20 of 20 tasks each, at the reporting floor — and success is near-perfect: only `tea` slips, to 89% on find-then-act, while the other three pass every run. +All four arms completed the full matrix — 20 of 20 tasks each — and success is near-perfect: only `tea` slips, to 85% overall (67% on find-then-act), while the other three pass every run. -Raw REST posts the lowest cost-equivalent tokens and leads every tier. -It is direct HTTP with the token in the request header, so it takes the fewest turns (4.3) and reads the least cached context, and no higher-level tool beats that on tokens alone. -This is the honest ceiling, and the reason gitea-axi does not claim the cost crown outright. +Raw REST posts the lowest cost-equivalent tokens, but only barely: gitea-axi lands within ~1% of it (17,815 vs 17,613), a gap well inside the noise of three trials. +Direct HTTP with the token in the request header is the token floor no wrapper is supposed to undercut — and a structured tool drawing level with it is the headline of this snapshot. -gitea-axi is a clear second overall and the cheapest of the structured tools: it undercuts the official `gitea-mcp` server and the `tea` CLI on every tier, at 100% success, with the lowest output-token count of any arm. -Note the split between raw and cost-equivalent tokens — gitea-axi spends more raw tokens than `gitea-mcp` yet costs less, because output is weighted 5× and gitea-axi's answers are compact. +gitea-axi is the cheapest structured interface by a wide margin — roughly 23% under both the official `gitea-mcp` server and the `tea` CLI — and it beats both of them on every tier, at 100% success. +It also takes the fewest turns of any arm (4.8, raw REST included) and the lowest output-token count of the shell arms; its compact TOON answers are what let a structured tool run this close to the floor. -By tier, raw REST's edge is widest on reads (10,921 vs gitea-axi's 14,415) — a read is one HTTP request for curl, where a CLI still spends a turn or two — and narrows on multi-step (24,348 vs 26,963), where the work itself dominates and interface overhead matters less. +By tier the picture is sharper than the overall total. +gitea-axi is the *cheapest arm outright* on the two discovery-heavy tiers — reads (13,294 vs raw's 14,656) and find-then-act (17,585 vs 18,631) — where finding the right entity is the work, and its compact search and list output beats reconstructing and parsing raw JSON. +Raw REST reclaims the lead on the mutation-heavy tiers — narrowly on single-mutation (15,212 vs 15,552), more clearly on multi-step (22,643 vs 26,079) — where the task is a handful of terse POSTs that no wrapper undercuts. +So the two trade tiers: gitea-axi wins where an interface earns its keep, raw REST wins where the request was already minimal. Cost parity on the scored suite also understates gitea-axi, because the suite is the subset every arm can do at all. The bonus table records capability-asymmetric operations — full-text issue search, rendering a PR's diff and checks, issue dependencies — that gitea-axi handles directly and raw REST has no first-class equivalent for. -_Snapshot: 2026-07-17 — 4 arms × 20 tasks × 3 trials each (240 samples), a single clean run with all four arms executed together against one live Gitea host; imputed cost is the mean per-task Anthropic-API-priced dollar cost._ +_Snapshot: 2026-07-19 — 4 arms × 20 tasks × 3 trials each (240 samples), a single clean run with all four arms executed together against one live Gitea host; imputed cost is the mean per-task Anthropic-API-priced dollar cost._ -- 2.47.3