feat: add benchmark seed provisioning (task 0025)
All checks were successful
CI / test (pull_request) Successful in 52s
CI / test (push) Successful in 49s

Add the deterministic, idempotent seed that brings a freshly provisioned
throwaway repository to a known ground truth before a trial runs, scripted
over the live Gitea API.

- bench/seed-plan.ts: the pure ground truth (fixed labels, an issue spread
  across the discriminating dimensions, and labelled/reviewed/real-branch
  pull requests) plus groundTruth(user), realizing it into a RepoState.
- bench/seed.ts: idempotent seeding reconciled by natural key, reusing
  gitea-axi's own tea-login credential discovery (no new secret handling).
- A live smoke tier (test:bench:smoke) validating the seed end-to-end and
  skipping cleanly when no host is configured, kept out of the deterministic
  bench tier.

Export selectLogin from src/context.ts so the bench reuses the exact
credential-selection path.
This commit was merged in pull request #26.
This commit is contained in:
2026-07-16 08:07:29 -04:00
parent 66f576a2b7
commit 8666c53557
11 changed files with 980 additions and 8 deletions

97
bench/seed.smoke.test.ts Normal file
View File

@@ -0,0 +1,97 @@
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import type { CliDeps } from "../src/deps.js";
import { SEED_PLAN } from "./seed-plan.js";
import {
deleteRepo,
provisionRepo,
readSeedSummary,
resolveBenchAccess,
seedRepo,
type BenchAccess,
type RepoCoords,
type SeedSummary,
} from "./seed.js";
/**
* The seed smoke tier: a single live validation that provisioning and seeding a
* throwaway repository really brings it to the declared ground truth, and that
* re-seeding is idempotent. The benchmark-harness spec designates seed
* provisioning as validated by a smoke run against a real host rather than by
* mocks, since its value is the real API interaction. Like the e2e tier keys off
* GITEA_AXI_E2E_URL, this suite skips cleanly when GITEA_AXI_BENCH_LOGIN is
* unset, which counts as a pass. Expected values are derived from SEED_PLAN, the
* declared ground-truth contract, never from seed.ts.
*/
const login = process.env.GITEA_AXI_BENCH_LOGIN;
describe.skipIf(!login)("seed smoke: provisioning and seeding", () => {
let access: BenchAccess;
let coords: RepoCoords;
beforeAll(async () => {
const deps: CliDeps = {
env: process.env,
cwd: process.cwd(),
globals: { login },
};
access = await resolveBenchAccess(deps, login!);
coords = await provisionRepo(access);
}, 180_000);
afterAll(async () => {
if (access && coords) {
await deleteRepo(access, coords).catch(() => {});
}
}, 60_000);
it("provisioning a fresh repository and seeding it produces the fixed labels, the open/closed issue spread, and the labelled and reviewed pull requests", async () => {
await seedRepo(access, coords);
const summary = await readSeedSummary(access, coords);
for (const label of SEED_PLAN.labels) {
expect(summary.labelNames).toContain(label.name);
}
const expectedOpen = new Set(
SEED_PLAN.issues.filter((i) => i.state === "open").map((i) => i.title),
);
const expectedClosed = new Set(
SEED_PLAN.issues.filter((i) => i.state === "closed").map((i) => i.title),
);
expect(new Set(summary.openIssueTitles)).toEqual(expectedOpen);
expect(new Set(summary.closedIssueTitles)).toEqual(expectedClosed);
expect(summary.selfAssignedIssueCount).toBe(
SEED_PLAN.issues.filter((i) => i.assignToSelf).length,
);
expect(summary.issuesWithCommentsCount).toBe(
SEED_PLAN.issues.filter((i) => i.comments.length > 0).length,
);
expect(new Set(summary.pullTitles)).toEqual(
new Set(SEED_PLAN.pullRequests.map((pr) => pr.title)),
);
const expectedLabeledPull = SEED_PLAN.pullRequests.find(
(pr) => pr.labels.length > 0,
);
expect(expectedLabeledPull).toBeDefined();
expect(summary.labeledPullTitles).toContain(expectedLabeledPull!.title);
const expectedReviewedPull = SEED_PLAN.pullRequests.find(
(pr) => pr.reviews.length > 0,
);
expect(expectedReviewedPull).toBeDefined();
expect(summary.reviewedPullTitles).toContain(expectedReviewedPull!.title);
});
it("re-running the seed against an already-seeded repository is idempotent", async () => {
await seedRepo(access, coords);
const first: SeedSummary = await readSeedSummary(access, coords);
await seedRepo(access, coords);
const second: SeedSummary = await readSeedSummary(access, coords);
expect(second).toEqual(first);
});
});