Files
gitea-axi/bench/seed.ts
alexion b0a0ffc1ab fix(bench): gate seeding on repo+index readiness and match read counts semantically
Two benchmark-harness races were being scored as agent failures. A freshly
seeded throwaway repo could 404 for an independent reader (the agent, a fresh
process) before Gitea made it consistent, and the async issue indexer lagged so
`search issues`/`search prs` returned nothing right after seeding — leaving the
agent unable to find the target it was asked to act on.

seedRepo now blocks until an independent read confirms the repo is reachable,
its full seeded issue and pull spread is visible, and a seeded issue and pull
are returned by the search index, before releasing the agent. It polls up to
60s and throws a loud harness error on timeout rather than letting a
propagation delay become a scored agent failure.

Also add an optional `pattern` regex to RequiredFact so a read answer's count is
recognised semantically rather than as a fixed phrase: "5 issues are currently
open" no longer fails against the literal "5 issues are open", while the
alphabetic-only filler run keeps a wrong count beside the right number (e.g.
"5 issues ... 3 open") failing.
2026-07-18 23:25:32 -04:00

567 lines
20 KiB
TypeScript

// Seed provisioning: bring a freshly provisioned throwaway repository to the
// deterministic ground truth of SEED_PLAN, scripted entirely over the Gitea API
// against the live host. This is the imperative boundary the seed-plan realizes;
// its value is the real API interaction, so it is validated by a smoke run
// (seed.smoke.test.ts) rather than mocked unit tests — see the benchmark-harness
// spec's testing decisions.
//
// Authentication reuses gitea-axi's own credential discovery (the tea login store
// and its git-credential token helper) rather than introducing new secret
// handling: resolveBenchAccess is a thin adapter over src/tea.ts and the login
// selection in src/context.ts.
//
// Every step is idempotent, keyed by a natural identity — a label by name, an
// issue or pull request by title, a comment or review by body, a branch by name —
// so re-running the seed against an already-seeded repository reconciles to the
// same ground truth instead of duplicating it.
import type { CliDeps } from "../src/deps.js";
import { selectLogin } from "../src/context.js";
import { getToken, listLogins } from "../src/tea.js";
import { groundTruth, SEED_PLAN, type SeedIssue, type SeedPullRequest } from "./seed-plan.js";
import type { RepoState, ReviewKind } from "./scoring-spec.js";
/** The live host coordinates a seeding run authenticates and talks to. */
export interface BenchAccess {
/** Gitea instance base URL, without the /api/v1 suffix. */
apiUrl: string;
token: string;
}
/** Owner and name of a throwaway repository on the host. */
export interface RepoCoords {
owner: string;
repo: string;
}
/**
* Resolve the host and token for a benchmark run by reusing gitea-axi's own
* credential discovery: list the tea logins, pick the named one exactly as the
* CLI does, and mint the token through tea's git-credential helper. No new secret
* handling is introduced — the benchmark rides the same path the product ships.
*/
export async function resolveBenchAccess(deps: CliDeps, loginName: string): Promise<BenchAccess> {
const logins = await listLogins(deps);
const login = selectLogin(logins, loginName, undefined);
const host = new URL(login.url).hostname;
const token = await getToken(deps, login, host);
return { apiUrl: login.url.replace(/\/+$/, ""), token };
}
/**
* One authenticated Gitea API round-trip; returns the raw response unchecked.
* Exported so the self-review probe (self-review.ts) can inspect a non-2xx
* response — a host that forbids self-approval — without it being thrown.
*/
export async function request(
access: BenchAccess,
method: string,
path: string,
payload?: unknown,
): Promise<Response> {
return fetch(`${access.apiUrl}/api/v1${path}`, {
method,
headers: {
authorization: `token ${access.token}`,
...(payload !== undefined ? { "content-type": "application/json" } : {}),
},
body: payload !== undefined ? JSON.stringify(payload) : undefined,
});
}
/** Fail on any non-2xx response, surfacing the method, path, status, and body. */
async function requireOk(res: Response, method: string, path: string): Promise<Response> {
if (!res.ok) {
throw new Error(`${method} ${path} failed (${res.status}): ${await res.text()}`);
}
return res;
}
/**
* Issue a request and require a 2xx, returning the parsed JSON body. Exported so
* the post-run snapshot capture (snapshot.ts) reads the live repository through
* the same authenticated round-trip the seed writes through.
*/
export async function send<T>(
access: BenchAccess,
method: string,
path: string,
payload?: unknown,
): Promise<T> {
const res = await requireOk(await request(access, method, path, payload), method, path);
return (await res.json()) as T;
}
/** The single available user: the account the token authenticates as. */
export async function currentUser(access: BenchAccess): Promise<string> {
const me = await send<{ login: string }>(access, "GET", "/user");
return me.login;
}
/**
* Create a fresh, private, auto-initialized throwaway repository under the
* authenticated user and return its coordinates. Each call mints a distinct name,
* so trials never collide.
*/
export async function provisionRepo(
access: BenchAccess,
name = `bench-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
): Promise<RepoCoords> {
const owner = await currentUser(access);
await send(access, "POST", "/user/repos", {
name,
auto_init: true,
default_branch: "main",
private: true,
});
return { owner, repo: name };
}
/**
* Best-effort deletion of a throwaway repository. `request` does not throw on a
* non-2xx, so a token lacking delete scope is ignored silently; the try/catch
* additionally tolerates a network-level failure. Cleanup must never fail a run.
*/
export async function deleteRepo(access: BenchAccess, coords: RepoCoords): Promise<void> {
try {
await request(access, "DELETE", `/repos/${coords.owner}/${coords.repo}`);
} catch {
// Swallow network-level failures; a non-2xx never reaches here.
}
}
/** Colours compare equal regardless of a leading `#` or letter case. */
function normalizeColor(color: string): string {
return color.replace(/^#/, "").toLowerCase();
}
interface GiteaLabel {
id: number;
name: string;
color: string;
description?: string;
}
/**
* Reconcile the repository's labels to the plan, keyed by name: create a missing
* label, patch one whose colour or description drifted, and leave a matching one
* untouched. Returns the name→id map the issue and pull-request steps need to
* apply labels.
*/
async function ensureLabels(access: BenchAccess, coords: RepoCoords): Promise<Map<string, number>> {
const base = `/repos/${coords.owner}/${coords.repo}/labels`;
const existing = await send<GiteaLabel[]>(access, "GET", `${base}?limit=100`);
const byName = new Map(existing.map((label) => [label.name, label]));
for (const label of SEED_PLAN.labels) {
const found = byName.get(label.name);
if (!found) {
const created = await send<GiteaLabel>(access, "POST", base, {
name: label.name,
color: label.color,
description: label.description ?? "",
});
byName.set(created.name, created);
} else if (
normalizeColor(found.color) !== normalizeColor(label.color) ||
(found.description ?? "") !== (label.description ?? "")
) {
await send<GiteaLabel>(access, "PATCH", `${base}/${found.id}`, {
color: label.color,
description: label.description ?? "",
});
}
}
return new Map([...byName].map(([name, label]) => [name, label.id]));
}
/** Replace an issue-or-pull-request's applied labels with exactly the given ids. */
async function applyLabels(
access: BenchAccess,
coords: RepoCoords,
number: number,
names: string[],
labelIds: Map<string, number>,
): Promise<void> {
const ids = names.map((name) => labelIds.get(name)).filter((id): id is number => id !== undefined);
await send(access, "PUT", `/repos/${coords.owner}/${coords.repo}/issues/${number}/labels`, {
labels: ids,
});
}
/**
* Add each item whose body is not already present at `base`, keyed by body. Both
* comments and reviews live at a single endpoint that lists and creates at the
* same path, so one reconciler serves them: it lists what is there, then posts
* only the items whose body is missing — which is what makes re-seeding a no-op.
*/
async function addMissingByBody<T>(
access: BenchAccess,
base: string,
items: T[],
bodyOf: (item: T) => string,
payloadOf: (item: T) => unknown,
): Promise<void> {
if (items.length === 0) {
return;
}
const existing = await send<{ body: string }[]>(access, "GET", base);
const present = new Set(existing.map((entry) => entry.body));
for (const item of items) {
if (!present.has(bodyOf(item))) {
await send(access, "POST", base, payloadOf(item));
}
}
}
/** Add each comment body not already present on an issue or pull request. */
async function ensureComments(
access: BenchAccess,
coords: RepoCoords,
number: number,
bodies: string[],
): Promise<void> {
await addMissingByBody(
access,
`/repos/${coords.owner}/${coords.repo}/issues/${number}/comments`,
bodies,
(body) => body,
(body) => ({ body }),
);
}
interface GiteaIssue {
number: number;
title: string;
}
/**
* Reconcile one plan issue, keyed by title: create it if absent, then declare its
* body, state, applied labels, and assignee presence (the single user or nobody)
* and add any missing comments. Every field is set to the desired value, so the
* step is idempotent whether the issue was just created or already seeded.
*/
async function ensureIssue(
access: BenchAccess,
coords: RepoCoords,
user: string,
issue: SeedIssue,
labelIds: Map<string, number>,
byTitle: Map<string, number>,
): Promise<void> {
const base = `/repos/${coords.owner}/${coords.repo}/issues`;
let number = byTitle.get(issue.title);
if (number === undefined) {
const created = await send<GiteaIssue>(access, "POST", base, {
title: issue.title,
body: issue.body,
});
number = created.number;
byTitle.set(issue.title, number);
}
await send(access, "PATCH", `${base}/${number}`, {
title: issue.title,
body: issue.body,
state: issue.state,
assignees: issue.assignToSelf ? [user] : [],
});
await applyLabels(access, coords, number, issue.labels, labelIds);
await ensureComments(access, coords, number, issue.comments);
}
/** The Gitea review event verb for each seed review kind. */
const REVIEW_EVENT: Record<ReviewKind, string> = {
comment: "COMMENT",
approved: "APPROVED",
"request-changes": "REQUEST_CHANGES",
};
/** Ensure the pull request's feature branch exists, creating it with its file. */
async function ensureBranch(
access: BenchAccess,
coords: RepoCoords,
pr: SeedPullRequest,
): Promise<void> {
const branchPath = `/repos/${coords.owner}/${coords.repo}/branches/${pr.headBranch}`;
const res = await request(access, "GET", branchPath);
if (res.ok) {
return;
}
if (res.status !== 404) {
await requireOk(res, "GET", branchPath);
}
await send(access, "POST", `/repos/${coords.owner}/${coords.repo}/contents/${pr.filePath}`, {
content: Buffer.from(pr.fileContent).toString("base64"),
message: `Seed ${pr.headBranch}`,
new_branch: pr.headBranch,
});
}
/** Add each review (matched by body) not already present on the pull request. */
async function ensureReviews(
access: BenchAccess,
coords: RepoCoords,
number: number,
reviews: SeedPullRequest["reviews"],
): Promise<void> {
await addMissingByBody(
access,
`/repos/${coords.owner}/${coords.repo}/pulls/${number}/reviews`,
reviews,
(review) => review.body,
(review) => ({ event: REVIEW_EVENT[review.kind], body: review.body }),
);
}
interface GiteaPull {
number: number;
title: string;
state?: string;
/** True once merged; a merged pull request cannot be reopened. */
merged?: boolean;
}
/**
* Reconcile one plan pull request, keyed by title: ensure its feature branch,
* open the pull request if absent, then declare its labels and add any missing
* comments and reviews. All content is authored by the single available user.
*/
async function ensurePullRequest(
access: BenchAccess,
coords: RepoCoords,
pr: SeedPullRequest,
labelIds: Map<string, number>,
byTitle: Map<string, GiteaPull>,
): Promise<void> {
await ensureBranch(access, coords, pr);
const base = `/repos/${coords.owner}/${coords.repo}/pulls`;
let number: number;
const existing = byTitle.get(pr.title);
if (existing === undefined) {
const created = await send<GiteaPull>(access, "POST", base, {
title: pr.title,
body: pr.body,
base: "main",
head: pr.headBranch,
});
number = created.number;
byTitle.set(pr.title, created);
} else {
number = existing.number;
// The ground truth declares every seeded pull request open. Reopen one that
// drifted closed (but never a merged one, which Gitea cannot reopen), so the
// seed reconciles state as declaratively as it does for issues.
if (existing.state === "closed" && existing.merged !== true) {
await send(access, "PATCH", `${base}/${number}`, { state: "open" });
}
}
await applyLabels(access, coords, number, pr.labels, labelIds);
await ensureComments(access, coords, number, pr.comments);
await ensureReviews(access, coords, number, pr.reviews);
}
/** How long the readiness gate polls before giving up, and how often it re-checks. */
const READINESS_TIMEOUT_MS = 60_000;
const READINESS_INTERVAL_MS = 750;
/** Resolve after `ms` milliseconds. */
function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** A search hit's minimal shape: the repository the matched issue/pull belongs to. */
interface SearchHit {
repository?: { name?: string } | null;
}
/**
* Whether the issue indexer has caught up: a full-text search for `title` of the
* given kind returns a hit in this repository. The endpoint spans every repo the
* owner can access (mirroring `gitea-axi search`), so a hit only counts when its
* repository matches. A non-2xx or a not-yet-indexed title reads as not-ready.
*/
async function searchIndexed(
access: BenchAccess,
coords: RepoCoords,
type: "issues" | "pulls",
title: string,
): Promise<boolean> {
const query = new URLSearchParams({ q: title, type, owner: coords.owner, state: "all", limit: "50" });
const res = await request(access, "GET", `/repos/issues/search?${query.toString()}`);
if (!res.ok) {
return false;
}
const hits = (await res.json()) as SearchHit[];
return hits.some((hit) => hit.repository?.name === coords.repo);
}
/**
* The reason a freshly seeded repository is not yet ready for the agent, or `null`
* when it is. Ready means an independent read sees the repository and its full
* seeded issue and pull spread, and the issue indexer returns a seeded issue and
* pull — the two consistency windows (repo visibility and index lag) an agent
* would otherwise race and fail against. Every read is non-throwing, so a
* transient error reads as not-ready and is retried rather than propagated.
*/
async function readinessGap(access: BenchAccess, coords: RepoCoords): Promise<string | null> {
const repoPath = `/repos/${coords.owner}/${coords.repo}`;
const repoRes = await request(access, "GET", repoPath);
if (!repoRes.ok) {
return `repository read returned ${repoRes.status}`;
}
const issuesRes = await request(access, "GET", `${repoPath}/issues?type=issues&state=all&limit=100`);
if (!issuesRes.ok) {
return `issue list returned ${issuesRes.status}`;
}
const issues = (await issuesRes.json()) as GiteaIssue[];
if (issues.length < SEED_PLAN.issues.length) {
return `only ${issues.length}/${SEED_PLAN.issues.length} issues visible`;
}
const pullsRes = await request(access, "GET", `${repoPath}/pulls?state=all&limit=100`);
if (!pullsRes.ok) {
return `pull list returned ${pullsRes.status}`;
}
const pulls = (await pullsRes.json()) as GiteaPull[];
if (pulls.length < SEED_PLAN.pullRequests.length) {
return `only ${pulls.length}/${SEED_PLAN.pullRequests.length} pull requests visible`;
}
const sampleIssue = SEED_PLAN.issues[0]!.title;
if (!(await searchIndexed(access, coords, "issues", sampleIssue))) {
return `issue "${sampleIssue}" not yet indexed for search`;
}
const samplePull = SEED_PLAN.pullRequests[0]!.title;
if (!(await searchIndexed(access, coords, "pulls", samplePull))) {
return `pull request "${samplePull}" not yet indexed for search`;
}
return null;
}
/**
* Block until a freshly seeded repository is fully consistent for the agent, or
* fail loudly if it never settles within the timeout. Gitea makes a just-created
* repository and its just-written issues and pulls visible to the seed's own
* writes immediately, but an independent reader — the agent, a fresh process
* moments later — can hit a brief 404 window on the repository and a longer lag on
* the async issue indexer. Both are the benchmark's races, not the tool's, so
* closing them here keeps a scored run measuring gitea-axi rather than host
* propagation. A timeout is a harness failure surfaced to the maintainer, never a
* scored agent failure.
*/
export async function waitForSeedReady(
access: BenchAccess,
coords: RepoCoords,
timeoutMs = READINESS_TIMEOUT_MS,
intervalMs = READINESS_INTERVAL_MS,
): Promise<void> {
const deadline = Date.now() + timeoutMs;
let gap = await readinessGap(access, coords);
while (gap !== null) {
if (Date.now() >= deadline) {
throw new Error(
`seeded repository ${coords.owner}/${coords.repo} not ready after ${timeoutMs}ms: ${gap}`,
);
}
await delay(intervalMs);
gap = await readinessGap(access, coords);
}
}
/**
* Seed a freshly provisioned repository to the ground truth, idempotently. Labels
* come first (so issues and pull requests can apply them), then the issue spread,
* then the pull requests. Returns the deterministic ground-truth RepoState the
* checker scores against; on a fresh repository the created numbers match it,
* and a re-run leaves them unchanged.
*
* Before returning, it waits out the repository-visibility and issue-indexer
* consistency windows (see waitForSeedReady), so the agent that runs next never
* races a repository that is not yet readable or searchable.
*/
export async function seedRepo(access: BenchAccess, coords: RepoCoords): Promise<RepoState> {
const user = await currentUser(access);
const labelIds = await ensureLabels(access, coords);
const issuesPath = `/repos/${coords.owner}/${coords.repo}/issues?type=issues&state=all&limit=100`;
const existingIssues = await send<GiteaIssue[]>(access, "GET", issuesPath);
const issuesByTitle = new Map(existingIssues.map((issue) => [issue.title, issue.number]));
for (const issue of SEED_PLAN.issues) {
await ensureIssue(access, coords, user, issue, labelIds, issuesByTitle);
}
const pullsPath = `/repos/${coords.owner}/${coords.repo}/pulls?state=all&limit=100`;
const existingPulls = await send<GiteaPull[]>(access, "GET", pullsPath);
const pullsByTitle = new Map(existingPulls.map((pull) => [pull.title, pull]));
for (const pr of SEED_PLAN.pullRequests) {
await ensurePullRequest(access, coords, pr, labelIds, pullsByTitle);
}
await waitForSeedReady(access, coords);
return groundTruth(user);
}
/**
* The observable facts the smoke run checks, read back from live Gitea (not from
* the plan) so the assertion compares the real repository against the declared
* ground truth rather than the plan against itself.
*/
export interface SeedSummary {
labelNames: string[];
openIssueTitles: string[];
closedIssueTitles: string[];
selfAssignedIssueCount: number;
issuesWithCommentsCount: number;
pullTitles: string[];
labeledPullTitles: string[];
reviewedPullTitles: string[];
}
interface GiteaIssueSummary {
title: string;
state: string;
assignees: { login: string }[] | null;
comments: number;
}
interface GiteaPullSummary {
number: number;
title: string;
labels: { name: string }[] | null;
}
/** Read the live repository into the summary the smoke run asserts against. */
export async function readSeedSummary(
access: BenchAccess,
coords: RepoCoords,
): Promise<SeedSummary> {
const repo = `/repos/${coords.owner}/${coords.repo}`;
const labels = await send<GiteaLabel[]>(access, "GET", `${repo}/labels?limit=100`);
const issues = await send<GiteaIssueSummary[]>(
access,
"GET",
`${repo}/issues?type=issues&state=all&limit=100`,
);
const pulls = await send<GiteaPullSummary[]>(access, "GET", `${repo}/pulls?state=all&limit=100`);
const reviewedPullTitles: string[] = [];
for (const pull of pulls) {
const reviews = await send<{ body: string }[]>(
access,
"GET",
`${repo}/pulls/${pull.number}/reviews`,
);
if (reviews.length > 0) {
reviewedPullTitles.push(pull.title);
}
}
return {
labelNames: labels.map((label) => label.name),
openIssueTitles: issues.filter((i) => i.state === "open").map((i) => i.title),
closedIssueTitles: issues.filter((i) => i.state === "closed").map((i) => i.title),
selfAssignedIssueCount: issues.filter((i) => (i.assignees ?? []).length > 0).length,
issuesWithCommentsCount: issues.filter((i) => i.comments > 0).length,
pullTitles: pulls.map((p) => p.title),
labeledPullTitles: pulls.filter((p) => (p.labels ?? []).length > 0).map((p) => p.title),
reviewedPullTitles,
};
}