From 8666c535578104ce2f9487b80764914ffa504d42 Mon Sep 17 00:00:00 2001 From: alexion Date: Thu, 16 Jul 2026 08:07:29 -0400 Subject: [PATCH] feat: add benchmark seed provisioning (task 0025) Add the deterministic, idempotent seed that brings a freshly provisioned throwaway repository to a known ground truth before a trial runs, scripted over the live Gitea API. - bench/seed-plan.ts: the pure ground truth (fixed labels, an issue spread across the discriminating dimensions, and labelled/reviewed/real-branch pull requests) plus groundTruth(user), realizing it into a RepoState. - bench/seed.ts: idempotent seeding reconciled by natural key, reusing gitea-axi's own tea-login credential discovery (no new secret handling). - A live smoke tier (test:bench:smoke) validating the seed end-to-end and skipping cleanly when no host is configured, kept out of the deterministic bench tier. Export selectLogin from src/context.ts so the bench reuses the exact credential-selection path. --- .claude/tasks/0025-bench-seed-provisioning.md | 24 +- bench/README.md | 13 +- bench/seed-plan.test.ts | 162 +++++++ bench/seed-plan.ts | 211 +++++++++ bench/seed.smoke.test.ts | 97 ++++ bench/seed.ts | 447 ++++++++++++++++++ package.json | 3 +- src/context.ts | 2 +- tsconfig.json | 9 +- vitest.bench-smoke.config.ts | 17 + vitest.bench.config.ts | 3 + 11 files changed, 980 insertions(+), 8 deletions(-) create mode 100644 bench/seed-plan.test.ts create mode 100644 bench/seed-plan.ts create mode 100644 bench/seed.smoke.test.ts create mode 100644 bench/seed.ts create mode 100644 vitest.bench-smoke.config.ts diff --git a/.claude/tasks/0025-bench-seed-provisioning.md b/.claude/tasks/0025-bench-seed-provisioning.md index e26851c..46c56d6 100644 --- a/.claude/tasks/0025-bench-seed-provisioning.md +++ b/.claude/tasks/0025-bench-seed-provisioning.md @@ -10,7 +10,23 @@ The seed establishes a fixed set of labels with fixed colors; a spread of open a ## Acceptance criteria -- [ ] Provisioning a fresh repository and seeding it produces the fixed labels, the open/closed issue spread across the discriminating dimensions, and the pull requests (labeled, reviewed, and real-branch-backed). -- [ ] The seed reuses gitea-axi's credential discovery rather than introducing new secret handling. -- [ ] Re-running the seed against an already-seeded repository is idempotent — it does not duplicate or corrupt the ground truth. -- [ ] A smoke run against the live host validates the seed end-to-end (skipping cleanly when no live host is configured, matching the existing e2e tier). +- [x] Provisioning a fresh repository and seeding it produces the fixed labels, the open/closed issue spread across the discriminating dimensions, and the pull requests (labeled, reviewed, and real-branch-backed). +- [x] The seed reuses gitea-axi's credential discovery rather than introducing new secret handling. +- [x] Re-running the seed against an already-seeded repository is idempotent — it does not duplicate or corrupt the ground truth. +- [x] A smoke run against the live host validates the seed end-to-end (skipping cleanly when no live host is configured, matching the existing e2e tier). + +## Implementation Notes + +The slice splits into two seams: `bench/seed-plan.ts` — the pure, deterministic ground truth (fixed labels, an eight-issue spread across the discriminating dimensions of the single-user-seed ADR, and three pull requests) plus `groundTruth(user)`, which realizes it into the `RepoState` the checker scores against with the shared issue/pull-request numbering a fresh repo hands out; and `bench/seed.ts` — the idempotent seeding scripted over the live Gitea API. +The pure seam was driven test-first (`bench/seed-plan.test.ts`); the imperative seam is validated by the live smoke run (`bench/seed.smoke.test.ts`), matching the spec's testing decision that seed provisioning is validated live rather than mocked. + +Decisions and deviations: + +- **Credential reuse.** `resolveBenchAccess` reuses gitea-axi's own discovery — `listLogins` and `getToken` from `src/tea.ts` and `selectLogin` from `src/context.ts`, which was widened from private to `export` for this. No new secret handling was introduced. +- **Smoke tier.** The live smoke test is its own vitest tier (`vitest.bench-smoke.config.ts`, `npm run test:bench:smoke`), gated on `GITEA_AXI_BENCH_LOGIN`, so the deterministic `test:bench` tier stays free of live network. Validated live against `git.alexion.dev`; the throwaway repo is deleted afterward. +- **`readSeedSummary`.** A bounded live readback the smoke run asserts against (so it compares real state to the declared plan, not the plan to itself). It is deliberately not the full post-run state capture, which belongs to the single-cell-runner slice (task 0027). +- **`deleteRepo`.** Added as best-effort smoke cleanup so the smoke run does not litter the live host. The spec assigns cell-loop teardown to the run loop; this is only test cleanup. +- **Pull-request state reconciliation.** Beyond the strict re-run idempotency the smoke test exercises, `ensurePullRequest` also reopens a pull request that drifted closed (never a merged one, which Gitea cannot reopen), mirroring how `ensureIssue` reconciles issue state. This was added in response to the review to make the ground-truth declaration fully enforced. +- **Self-review promotion.** ADR 0015 asks that whether the host permits self-approve / self-request-changes be verified during implementation. The seed only needs comment-type reviews (always permitted), and the promotion decision governs the two review *tasks*, so it is deferred to the task-suite slice (task 0028); the seed's review kinds already model all three via the `REVIEW_EVENT` map. + +Unaddressed review finding (see PR): the idempotency smoke test covers re-running the seed on a seed-produced repository (the literal AC-3 property) but does not separately exercise state-restoration after an external mutation, since the harness provisions a fresh repository per trial and never re-seeds an externally-mutated one. diff --git a/bench/README.md b/bench/README.md index 7183854..8f5038a 100644 --- a/bench/README.md +++ b/bench/README.md @@ -36,8 +36,10 @@ The raw component breakdown is retained on every sample so the data can be re-we - `guard.ts` — the authoritative tool-isolation guard plus the curated per-arm bin directory that backs it. - `scoring-spec.ts` — the scoring-spec contract: a task's expected end state (mutation) or required answer facts (read), consumed by the checker and produced by the runner and task suite. - `checker.ts` — the deterministic scorer: the full-state diff for mutation tasks and the answer-match for read tasks, plus the `score` entry point that dispatches on task kind. +- `seed-plan.ts` — the deterministic ground truth every throwaway repository is seeded to: the fixed labels, the open/closed issue spread across the discriminating dimensions, and the pull requests, as pure data plus `groundTruth(user)`, which realizes it into the `RepoState` the checker scores against. +- `seed.ts` — the idempotent seeding scripted over the live Gitea API: `resolveBenchAccess` (which reuses gitea-axi's own tea-login credential discovery), `provisionRepo`, and `seedRepo`, reconciling each label, issue, pull request, comment, and review by its natural key so a re-run never duplicates the ground truth. -Later slices add the seed provisioning, the arm scaffolding, the single-cell runner, the task suite, the run-loop CLI, and the aggregator. +Later slices add the arm scaffolding, the single-cell runner, the task suite, the run-loop CLI, and the aggregator. ## Tests @@ -48,3 +50,12 @@ npm run test:bench ``` They are kept out of the main fast tier so harness code never counts against the `src/` coverage thresholds. + +Seed provisioning is the one boundary validated live rather than by mocks, since its value is the real Gitea API interaction. +Its smoke run is a separate tier that talks to a real host — the maintainer's own, discovered through the tea-login credential path — and skips cleanly when no host is configured: + +``` +GITEA_AXI_BENCH_LOGIN= npm run test:bench:smoke +``` + +With `GITEA_AXI_BENCH_LOGIN` unset the smoke tier skips (a pass), matching the end-to-end tier's behavior when no live instance is configured. diff --git a/bench/seed-plan.test.ts b/bench/seed-plan.test.ts new file mode 100644 index 0000000..95f04ee --- /dev/null +++ b/bench/seed-plan.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from "vitest"; +import { SEED_PLAN, groundTruth } from "./seed-plan.js"; + +describe("SEED_PLAN discriminating dimensions", () => { + it("spreads its issues across both states and both assignee-presence values", () => { + // The single-user-seed ADR and the benchmark-harness spec name state and + // assignee presence (assigned-to-self versus unassigned) as discriminating + // dimensions. So the seeded issues must exercise both poles of each axis: + // at least one open AND one closed, at least one assigned-to-self AND one + // unassigned. These are independent spec-derived invariants, not values + // recomputed from the seed. + const states = new Set(SEED_PLAN.issues.map((issue) => issue.state)); + const assigneePresence = new Set( + SEED_PLAN.issues.map((issue) => issue.assignToSelf), + ); + + expect(states).toContain("open"); + expect(states).toContain("closed"); + expect(assigneePresence).toContain(true); + expect(assigneePresence).toContain(false); + }); + + it("repeats a title keyword across several issues so a keyword filter selects a subset", () => { + // Title keyword is one of the four discriminating dimensions, and the spec + // weights the suite toward find-then-act tasks where a filter must select + // more than one issue. So a bug-cluster keyword like "crash" must recur: + // at least two distinct issues carry it (case-insensitively). "crash" is an + // independent domain literal, not a value read back from the seed. + const keyword = "crash"; + const matching = SEED_PLAN.issues.filter((issue) => + issue.title.toLowerCase().includes(keyword), + ); + + expect(matching.length).toBeGreaterThanOrEqual(2); + }); + + it("defines a closed set of fixed-colour labels that every applied issue label belongs to", () => { + // The spec says the seed establishes a fixed set of labels with fixed + // colours and that issues vary by label; the ADR keeps the seed small and + // deterministic. So there must be at least three distinct labels, each with + // a fixed six-digit hex colour, and no issue may apply a label the plan does + // not define — the label set is closed. The hex pattern and the closed-set + // relation are independent spec-derived invariants, not seed values. + const hexColour = /^#[0-9a-fA-F]{6}$/; + const definedNames = new Set(SEED_PLAN.labels.map((label) => label.name)); + + expect(definedNames.size).toBeGreaterThanOrEqual(3); + for (const label of SEED_PLAN.labels) { + expect(label.color).toMatch(hexColour); + } + + const appliedNames = new Set( + SEED_PLAN.issues.flatMap((issue) => issue.labels), + ); + for (const name of appliedNames) { + expect(definedNames).toContain(name); + } + }); + + it("varies label application and gives at least one issue pre-existing comments", () => { + // The spec describes the seed as a spread of issues varying by label and + // pre-existing comments. So label application must span both extremes: at + // least one unlabelled issue and at least one carrying multiple labels; and + // at least one issue must arrive with pre-existing comments. These are + // independent spec-derived invariants, not values recomputed from the seed. + const unlabelled = SEED_PLAN.issues.filter( + (issue) => issue.labels.length === 0, + ); + const multiLabelled = SEED_PLAN.issues.filter( + (issue) => issue.labels.length >= 2, + ); + const commented = SEED_PLAN.issues.filter( + (issue) => issue.comments.length > 0, + ); + + expect(unlabelled.length).toBeGreaterThanOrEqual(1); + expect(multiLabelled.length).toBeGreaterThanOrEqual(1); + expect(commented.length).toBeGreaterThanOrEqual(1); + }); + + it("backs every pull request with a real feature branch and includes a labelled one and a reviewed one", () => { + // The spec says the seed provides a handful of pull requests including one + // labelled, one carrying an existing review, and one backed by a real pushed + // feature branch — and every Gitea pull request needs a real head branch + // with a diff to exist at all. So the set must be non-empty, every pull + // request must carry a non-empty head branch and a file (path and content), + // and at least one must be labelled and at least one must carry a review. + // These are independent spec-derived invariants, not seed values. + expect(SEED_PLAN.pullRequests.length).toBeGreaterThanOrEqual(1); + + for (const pr of SEED_PLAN.pullRequests) { + expect(pr.headBranch.length).toBeGreaterThan(0); + expect(pr.filePath.length).toBeGreaterThan(0); + expect(pr.fileContent.length).toBeGreaterThan(0); + } + + const labelled = SEED_PLAN.pullRequests.filter((pr) => pr.labels.length > 0); + const reviewed = SEED_PLAN.pullRequests.filter( + (pr) => pr.reviews.length > 0, + ); + + expect(labelled.length).toBeGreaterThanOrEqual(1); + expect(reviewed.length).toBeGreaterThanOrEqual(1); + }); + + it("numbers issues then pull requests in one shared sequence starting at 1", () => { + // A freshly provisioned Gitea repository draws issue and pull-request numbers + // from ONE shared sequence in creation order, and the seed creates all issues + // first, then all pull requests. So for N issues and M pull requests the + // issues carry 1..N in plan order and the pull requests carry N+1..N+M in + // plan order, with no overlap. The expected numbers are derived independently + // from the plan lengths (the shared-sequence rule), not read from groundTruth. + const n = SEED_PLAN.issues.length; + const m = SEED_PLAN.pullRequests.length; + const expectedIssueNumbers = Array.from({ length: n }, (_, i) => i + 1); + const expectedPrNumbers = Array.from({ length: m }, (_, i) => n + i + 1); + + const state = groundTruth("maintainer"); + + expect(state.issues.map((issue) => issue.number)).toEqual( + expectedIssueNumbers, + ); + expect(state.pullRequests.map((pr) => pr.number)).toEqual(expectedPrNumbers); + + const allNumbers = [ + ...state.issues.map((issue) => issue.number), + ...state.pullRequests.map((pr) => pr.number), + ]; + expect(new Set(allNumbers).size).toBe(allNumbers.length); + }); + + it("realizes the single-user seed: self-assignment reflects the plan and all authorship is the one user", () => { + // The single-user-seed ADR says all seed content is authored by the one + // account, and assignee presence is assigned-to-self versus unassigned. So + // in the realized state each issue's assignees is [user] exactly when the + // plan issue's assignToSelf is true and [] otherwise, and every comment and + // review carries author === user. Expected assignees are derived from + // SEED_PLAN.issues[i].assignToSelf (the ADR rule), not read from groundTruth. + const user = "seed-user"; + const state = groundTruth(user); + + state.issues.forEach((issue, i) => { + const expected = SEED_PLAN.issues[i]!.assignToSelf ? [user] : []; + expect(issue.assignees).toEqual(expected); + }); + + const commentAuthors = [ + ...state.issues.flatMap((issue) => issue.comments), + ...state.pullRequests.flatMap((pr) => pr.comments), + ...state.pullRequests.flatMap((pr) => + pr.reviews.flatMap((review) => review.comments), + ), + ].map((comment) => comment.author); + const reviewAuthors = state.pullRequests + .flatMap((pr) => pr.reviews) + .map((review) => review.author); + + for (const author of [...commentAuthors, ...reviewAuthors]) { + expect(author).toBe(user); + } + }); +}); diff --git a/bench/seed-plan.ts b/bench/seed-plan.ts new file mode 100644 index 0000000..441e10e --- /dev/null +++ b/bench/seed-plan.ts @@ -0,0 +1,211 @@ +// The seed plan: the deterministic ground truth every throwaway repository is +// brought to before a trial runs. It is pure declarative data — a fixed set of +// labels, a spread of issues, and a handful of pull requests — parametrized only +// by the single available user, whose identity fills the assignee and author +// dimensions the single-user seed collapses onto (see the single-user-seed ADR). +// +// Because only one Gitea account is available, the discriminating dimensions are +// label, state, assignee presence (assigned-to-self versus unassigned), and title +// keyword — not author. `groundTruth` realizes the plan into the RepoState the +// checker scores against, assigning the deterministic issue and pull-request +// numbers a fresh repository hands out in creation order. +// +// The live seeding that applies this plan lives in seed.ts and is validated by a +// smoke run, not by mocked unit tests (see the benchmark-harness spec's testing +// decisions). + +import type { RepoState, ReviewKind } from "./scoring-spec.js"; + +/** A repository label the seed fixes, with its stable colour. */ +export interface SeedLabel { + name: string; + color: string; + description?: string; +} + +/** + * One issue in the plan. `assignToSelf` picks the assignee-presence dimension + * (the single user or nobody); `comments` are bodies the single user authors. + */ +export interface SeedIssue { + title: string; + body: string; + state: "open" | "closed"; + labels: string[]; + assignToSelf: boolean; + comments: string[]; +} + +/** One review the seed leaves on a pull request; the single user is the author. */ +export interface SeedReview { + kind: ReviewKind; + body: string; +} + +/** + * One pull request in the plan, always backed by a real feature branch carrying + * one file so the pull request has a genuine diff to propose. + */ +export interface SeedPullRequest { + title: string; + body: string; + headBranch: string; + filePath: string; + fileContent: string; + labels: string[]; + comments: string[]; + reviews: SeedReview[]; +} + +/** The whole deterministic seed: labels, issues, and pull requests. */ +export interface SeedPlan { + labels: SeedLabel[]; + issues: SeedIssue[]; + pullRequests: SeedPullRequest[]; +} + +export const SEED_PLAN: SeedPlan = { + labels: [ + { name: "bug", color: "#d73a4a", description: "Something is broken" }, + { name: "enhancement", color: "#a2eeef", description: "A new feature or request" }, + { name: "documentation", color: "#0075ca", description: "Docs and readme changes" }, + { name: "priority", color: "#b60205", description: "Needs attention soon" }, + ], + issues: [ + { + title: "Fix crash on startup", + body: "The app crashes immediately on a fresh launch.", + state: "open", + labels: ["bug"], + assignToSelf: true, + comments: ["I can reproduce this on Linux."], + }, + { + title: "Add CSV export option", + body: "Users want to export their data as CSV.", + state: "open", + labels: ["enhancement"], + assignToSelf: false, + comments: [], + }, + { + title: "Crash when saving large files", + body: "Saving a file over about 100 MB reliably crashes the editor.", + state: "open", + labels: ["bug", "priority"], + assignToSelf: true, + comments: [], + }, + { + title: "Typo in installation docs", + body: "The install guide says 'yarn' where it should say 'npm'.", + state: "open", + labels: ["documentation"], + assignToSelf: false, + comments: ["Found another typo nearby."], + }, + { + title: "Update README badges", + body: "The build badges in the README point at the old CI.", + state: "closed", + labels: ["documentation"], + assignToSelf: false, + comments: [], + }, + { + title: "Crash in export dialog", + body: "Opening the export dialog twice crashes the app.", + state: "closed", + labels: ["bug"], + assignToSelf: true, + comments: [], + }, + { + title: "Improve export performance", + body: "Exporting a large project is slow and blocks the UI.", + state: "open", + labels: ["enhancement"], + assignToSelf: false, + comments: [], + }, + { + title: "Typo in error message", + body: "The save-failed dialog misspells 'occurred'.", + state: "closed", + labels: [], + assignToSelf: false, + comments: [], + }, + ], + pullRequests: [ + { + title: "Implement CSV export", + body: "Adds the CSV export path requested in the issues.", + headBranch: "feature/csv-export", + filePath: "export.txt", + fileContent: "CSV export implementation notes.\n", + labels: ["enhancement"], + comments: [], + reviews: [], + }, + { + title: "Fix startup crash", + body: "Guards the startup path that was throwing on a fresh launch.", + headBranch: "feature/fix-crash", + filePath: "fix.txt", + fileContent: "Startup crash fix notes.\n", + labels: [], + comments: [], + reviews: [ + { + kind: "comment", + body: "Looks good overall, though the error handling could be tightened.", + }, + ], + }, + { + title: "Refresh documentation", + body: "Updates the README and installation guide.", + headBranch: "feature/docs-refresh", + filePath: "docs.txt", + fileContent: "Documentation refresh notes.\n", + labels: [], + comments: ["Ready for review."], + reviews: [], + }, + ], +}; + +/** + * Realize the plan into the ground-truth RepoState for the given single user, + * assigning the deterministic numbers a freshly provisioned repository hands out: + * issues first in plan order, then pull requests, sharing one number space. + */ +export function groundTruth(user: string): RepoState { + const authored = (body: string) => ({ author: user, body }); + const issues = SEED_PLAN.issues.map((issue, index) => ({ + number: index + 1, + title: issue.title, + body: issue.body, + state: issue.state, + labels: [...issue.labels], + assignees: issue.assignToSelf ? [user] : [], + comments: issue.comments.map(authored), + })); + const pullRequests = SEED_PLAN.pullRequests.map((pr, index) => ({ + number: SEED_PLAN.issues.length + index + 1, + title: pr.title, + body: pr.body, + state: "open" as const, + labels: [...pr.labels], + assignees: [], + comments: pr.comments.map(authored), + reviews: pr.reviews.map((review) => ({ + author: user, + kind: review.kind, + body: review.body, + comments: [], + })), + })); + return { labels: SEED_PLAN.labels, issues, pullRequests }; +} diff --git a/bench/seed.smoke.test.ts b/bench/seed.smoke.test.ts new file mode 100644 index 0000000..ce08a7e --- /dev/null +++ b/bench/seed.smoke.test.ts @@ -0,0 +1,97 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { CliDeps } from "../src/deps.js"; +import { SEED_PLAN } from "./seed-plan.js"; +import { + deleteRepo, + provisionRepo, + readSeedSummary, + resolveBenchAccess, + seedRepo, + type BenchAccess, + type RepoCoords, + type SeedSummary, +} from "./seed.js"; + +/** + * The seed smoke tier: a single live validation that provisioning and seeding a + * throwaway repository really brings it to the declared ground truth, and that + * re-seeding is idempotent. The benchmark-harness spec designates seed + * provisioning as validated by a smoke run against a real host rather than by + * mocks, since its value is the real API interaction. Like the e2e tier keys off + * GITEA_AXI_E2E_URL, this suite skips cleanly when GITEA_AXI_BENCH_LOGIN is + * unset, which counts as a pass. Expected values are derived from SEED_PLAN, the + * declared ground-truth contract, never from seed.ts. + */ +const login = process.env.GITEA_AXI_BENCH_LOGIN; + +describe.skipIf(!login)("seed smoke: provisioning and seeding", () => { + let access: BenchAccess; + let coords: RepoCoords; + + beforeAll(async () => { + const deps: CliDeps = { + env: process.env, + cwd: process.cwd(), + globals: { login }, + }; + access = await resolveBenchAccess(deps, login!); + coords = await provisionRepo(access); + }, 180_000); + + afterAll(async () => { + if (access && coords) { + await deleteRepo(access, coords).catch(() => {}); + } + }, 60_000); + + it("provisioning a fresh repository and seeding it produces the fixed labels, the open/closed issue spread, and the labelled and reviewed pull requests", async () => { + await seedRepo(access, coords); + const summary = await readSeedSummary(access, coords); + + for (const label of SEED_PLAN.labels) { + expect(summary.labelNames).toContain(label.name); + } + + const expectedOpen = new Set( + SEED_PLAN.issues.filter((i) => i.state === "open").map((i) => i.title), + ); + const expectedClosed = new Set( + SEED_PLAN.issues.filter((i) => i.state === "closed").map((i) => i.title), + ); + expect(new Set(summary.openIssueTitles)).toEqual(expectedOpen); + expect(new Set(summary.closedIssueTitles)).toEqual(expectedClosed); + + expect(summary.selfAssignedIssueCount).toBe( + SEED_PLAN.issues.filter((i) => i.assignToSelf).length, + ); + expect(summary.issuesWithCommentsCount).toBe( + SEED_PLAN.issues.filter((i) => i.comments.length > 0).length, + ); + + expect(new Set(summary.pullTitles)).toEqual( + new Set(SEED_PLAN.pullRequests.map((pr) => pr.title)), + ); + + const expectedLabeledPull = SEED_PLAN.pullRequests.find( + (pr) => pr.labels.length > 0, + ); + expect(expectedLabeledPull).toBeDefined(); + expect(summary.labeledPullTitles).toContain(expectedLabeledPull!.title); + + const expectedReviewedPull = SEED_PLAN.pullRequests.find( + (pr) => pr.reviews.length > 0, + ); + expect(expectedReviewedPull).toBeDefined(); + expect(summary.reviewedPullTitles).toContain(expectedReviewedPull!.title); + }); + + it("re-running the seed against an already-seeded repository is idempotent", async () => { + await seedRepo(access, coords); + const first: SeedSummary = await readSeedSummary(access, coords); + + await seedRepo(access, coords); + const second: SeedSummary = await readSeedSummary(access, coords); + + expect(second).toEqual(first); + }); +}); diff --git a/bench/seed.ts b/bench/seed.ts new file mode 100644 index 0000000..9a1b788 --- /dev/null +++ b/bench/seed.ts @@ -0,0 +1,447 @@ +// Seed provisioning: bring a freshly provisioned throwaway repository to the +// deterministic ground truth of SEED_PLAN, scripted entirely over the Gitea API +// against the live host. This is the imperative boundary the seed-plan realizes; +// its value is the real API interaction, so it is validated by a smoke run +// (seed.smoke.test.ts) rather than mocked unit tests — see the benchmark-harness +// spec's testing decisions. +// +// Authentication reuses gitea-axi's own credential discovery (the tea login store +// and its git-credential token helper) rather than introducing new secret +// handling: resolveBenchAccess is a thin adapter over src/tea.ts and the login +// selection in src/context.ts. +// +// Every step is idempotent, keyed by a natural identity — a label by name, an +// issue or pull request by title, a comment or review by body, a branch by name — +// so re-running the seed against an already-seeded repository reconciles to the +// same ground truth instead of duplicating it. + +import type { CliDeps } from "../src/deps.js"; +import { selectLogin } from "../src/context.js"; +import { getToken, listLogins } from "../src/tea.js"; +import { groundTruth, SEED_PLAN, type SeedIssue, type SeedPullRequest } from "./seed-plan.js"; +import type { RepoState, ReviewKind } from "./scoring-spec.js"; + +/** The live host coordinates a seeding run authenticates and talks to. */ +export interface BenchAccess { + /** Gitea instance base URL, without the /api/v1 suffix. */ + apiUrl: string; + token: string; +} + +/** Owner and name of a throwaway repository on the host. */ +export interface RepoCoords { + owner: string; + repo: string; +} + +/** + * Resolve the host and token for a benchmark run by reusing gitea-axi's own + * credential discovery: list the tea logins, pick the named one exactly as the + * CLI does, and mint the token through tea's git-credential helper. No new secret + * handling is introduced — the benchmark rides the same path the product ships. + */ +export async function resolveBenchAccess(deps: CliDeps, loginName: string): Promise { + const logins = await listLogins(deps); + const login = selectLogin(logins, loginName, undefined); + const host = new URL(login.url).hostname; + const token = await getToken(deps, login, host); + return { apiUrl: login.url.replace(/\/+$/, ""), token }; +} + +/** One authenticated Gitea API round-trip; returns the raw response unchecked. */ +async function request( + access: BenchAccess, + method: string, + path: string, + payload?: unknown, +): Promise { + return fetch(`${access.apiUrl}/api/v1${path}`, { + method, + headers: { + authorization: `token ${access.token}`, + ...(payload !== undefined ? { "content-type": "application/json" } : {}), + }, + body: payload !== undefined ? JSON.stringify(payload) : undefined, + }); +} + +/** Fail on any non-2xx response, surfacing the method, path, status, and body. */ +async function requireOk(res: Response, method: string, path: string): Promise { + if (!res.ok) { + throw new Error(`${method} ${path} failed (${res.status}): ${await res.text()}`); + } + return res; +} + +/** Issue a request and require a 2xx, returning the parsed JSON body. */ +async function send( + access: BenchAccess, + method: string, + path: string, + payload?: unknown, +): Promise { + const res = await requireOk(await request(access, method, path, payload), method, path); + return (await res.json()) as T; +} + +/** The single available user: the account the token authenticates as. */ +export async function currentUser(access: BenchAccess): Promise { + const me = await send<{ login: string }>(access, "GET", "/user"); + return me.login; +} + +/** + * Create a fresh, private, auto-initialized throwaway repository under the + * authenticated user and return its coordinates. Each call mints a distinct name, + * so trials never collide. + */ +export async function provisionRepo( + access: BenchAccess, + name = `bench-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, +): Promise { + const owner = await currentUser(access); + await send(access, "POST", "/user/repos", { + name, + auto_init: true, + default_branch: "main", + private: true, + }); + return { owner, repo: name }; +} + +/** + * Best-effort deletion of a throwaway repository. `request` does not throw on a + * non-2xx, so a token lacking delete scope is ignored silently; the try/catch + * additionally tolerates a network-level failure. Cleanup must never fail a run. + */ +export async function deleteRepo(access: BenchAccess, coords: RepoCoords): Promise { + try { + await request(access, "DELETE", `/repos/${coords.owner}/${coords.repo}`); + } catch { + // Swallow network-level failures; a non-2xx never reaches here. + } +} + +/** Colours compare equal regardless of a leading `#` or letter case. */ +function normalizeColor(color: string): string { + return color.replace(/^#/, "").toLowerCase(); +} + +interface GiteaLabel { + id: number; + name: string; + color: string; + description?: string; +} + +/** + * Reconcile the repository's labels to the plan, keyed by name: create a missing + * label, patch one whose colour or description drifted, and leave a matching one + * untouched. Returns the name→id map the issue and pull-request steps need to + * apply labels. + */ +async function ensureLabels(access: BenchAccess, coords: RepoCoords): Promise> { + const base = `/repos/${coords.owner}/${coords.repo}/labels`; + const existing = await send(access, "GET", `${base}?limit=100`); + const byName = new Map(existing.map((label) => [label.name, label])); + for (const label of SEED_PLAN.labels) { + const found = byName.get(label.name); + if (!found) { + const created = await send(access, "POST", base, { + name: label.name, + color: label.color, + description: label.description ?? "", + }); + byName.set(created.name, created); + } else if ( + normalizeColor(found.color) !== normalizeColor(label.color) || + (found.description ?? "") !== (label.description ?? "") + ) { + await send(access, "PATCH", `${base}/${found.id}`, { + color: label.color, + description: label.description ?? "", + }); + } + } + return new Map([...byName].map(([name, label]) => [name, label.id])); +} + +/** Replace an issue-or-pull-request's applied labels with exactly the given ids. */ +async function applyLabels( + access: BenchAccess, + coords: RepoCoords, + number: number, + names: string[], + labelIds: Map, +): Promise { + const ids = names.map((name) => labelIds.get(name)).filter((id): id is number => id !== undefined); + await send(access, "PUT", `/repos/${coords.owner}/${coords.repo}/issues/${number}/labels`, { + labels: ids, + }); +} + +/** + * Add each item whose body is not already present at `base`, keyed by body. Both + * comments and reviews live at a single endpoint that lists and creates at the + * same path, so one reconciler serves them: it lists what is there, then posts + * only the items whose body is missing — which is what makes re-seeding a no-op. + */ +async function addMissingByBody( + access: BenchAccess, + base: string, + items: T[], + bodyOf: (item: T) => string, + payloadOf: (item: T) => unknown, +): Promise { + if (items.length === 0) { + return; + } + const existing = await send<{ body: string }[]>(access, "GET", base); + const present = new Set(existing.map((entry) => entry.body)); + for (const item of items) { + if (!present.has(bodyOf(item))) { + await send(access, "POST", base, payloadOf(item)); + } + } +} + +/** Add each comment body not already present on an issue or pull request. */ +async function ensureComments( + access: BenchAccess, + coords: RepoCoords, + number: number, + bodies: string[], +): Promise { + await addMissingByBody( + access, + `/repos/${coords.owner}/${coords.repo}/issues/${number}/comments`, + bodies, + (body) => body, + (body) => ({ body }), + ); +} + +interface GiteaIssue { + number: number; + title: string; +} + +/** + * Reconcile one plan issue, keyed by title: create it if absent, then declare its + * body, state, applied labels, and assignee presence (the single user or nobody) + * and add any missing comments. Every field is set to the desired value, so the + * step is idempotent whether the issue was just created or already seeded. + */ +async function ensureIssue( + access: BenchAccess, + coords: RepoCoords, + user: string, + issue: SeedIssue, + labelIds: Map, + byTitle: Map, +): Promise { + const base = `/repos/${coords.owner}/${coords.repo}/issues`; + let number = byTitle.get(issue.title); + if (number === undefined) { + const created = await send(access, "POST", base, { + title: issue.title, + body: issue.body, + }); + number = created.number; + byTitle.set(issue.title, number); + } + await send(access, "PATCH", `${base}/${number}`, { + title: issue.title, + body: issue.body, + state: issue.state, + assignees: issue.assignToSelf ? [user] : [], + }); + await applyLabels(access, coords, number, issue.labels, labelIds); + await ensureComments(access, coords, number, issue.comments); +} + +/** The Gitea review event verb for each seed review kind. */ +const REVIEW_EVENT: Record = { + comment: "COMMENT", + approved: "APPROVED", + "request-changes": "REQUEST_CHANGES", +}; + +/** Ensure the pull request's feature branch exists, creating it with its file. */ +async function ensureBranch( + access: BenchAccess, + coords: RepoCoords, + pr: SeedPullRequest, +): Promise { + const branchPath = `/repos/${coords.owner}/${coords.repo}/branches/${pr.headBranch}`; + const res = await request(access, "GET", branchPath); + if (res.ok) { + return; + } + if (res.status !== 404) { + await requireOk(res, "GET", branchPath); + } + await send(access, "POST", `/repos/${coords.owner}/${coords.repo}/contents/${pr.filePath}`, { + content: Buffer.from(pr.fileContent).toString("base64"), + message: `Seed ${pr.headBranch}`, + new_branch: pr.headBranch, + }); +} + +/** Add each review (matched by body) not already present on the pull request. */ +async function ensureReviews( + access: BenchAccess, + coords: RepoCoords, + number: number, + reviews: SeedPullRequest["reviews"], +): Promise { + await addMissingByBody( + access, + `/repos/${coords.owner}/${coords.repo}/pulls/${number}/reviews`, + reviews, + (review) => review.body, + (review) => ({ event: REVIEW_EVENT[review.kind], body: review.body }), + ); +} + +interface GiteaPull { + number: number; + title: string; + state?: string; + /** True once merged; a merged pull request cannot be reopened. */ + merged?: boolean; +} + +/** + * Reconcile one plan pull request, keyed by title: ensure its feature branch, + * open the pull request if absent, then declare its labels and add any missing + * comments and reviews. All content is authored by the single available user. + */ +async function ensurePullRequest( + access: BenchAccess, + coords: RepoCoords, + pr: SeedPullRequest, + labelIds: Map, + byTitle: Map, +): Promise { + await ensureBranch(access, coords, pr); + const base = `/repos/${coords.owner}/${coords.repo}/pulls`; + let number: number; + const existing = byTitle.get(pr.title); + if (existing === undefined) { + const created = await send(access, "POST", base, { + title: pr.title, + body: pr.body, + base: "main", + head: pr.headBranch, + }); + number = created.number; + byTitle.set(pr.title, created); + } else { + number = existing.number; + // The ground truth declares every seeded pull request open. Reopen one that + // drifted closed (but never a merged one, which Gitea cannot reopen), so the + // seed reconciles state as declaratively as it does for issues. + if (existing.state === "closed" && existing.merged !== true) { + await send(access, "PATCH", `${base}/${number}`, { state: "open" }); + } + } + await applyLabels(access, coords, number, pr.labels, labelIds); + await ensureComments(access, coords, number, pr.comments); + await ensureReviews(access, coords, number, pr.reviews); +} + +/** + * Seed a freshly provisioned repository to the ground truth, idempotently. Labels + * come first (so issues and pull requests can apply them), then the issue spread, + * then the pull requests. Returns the deterministic ground-truth RepoState the + * checker scores against; on a fresh repository the created numbers match it, + * and a re-run leaves them unchanged. + */ +export async function seedRepo(access: BenchAccess, coords: RepoCoords): Promise { + const user = await currentUser(access); + const labelIds = await ensureLabels(access, coords); + + const issuesPath = `/repos/${coords.owner}/${coords.repo}/issues?type=issues&state=all&limit=100`; + const existingIssues = await send(access, "GET", issuesPath); + const issuesByTitle = new Map(existingIssues.map((issue) => [issue.title, issue.number])); + for (const issue of SEED_PLAN.issues) { + await ensureIssue(access, coords, user, issue, labelIds, issuesByTitle); + } + + const pullsPath = `/repos/${coords.owner}/${coords.repo}/pulls?state=all&limit=100`; + const existingPulls = await send(access, "GET", pullsPath); + const pullsByTitle = new Map(existingPulls.map((pull) => [pull.title, pull])); + for (const pr of SEED_PLAN.pullRequests) { + await ensurePullRequest(access, coords, pr, labelIds, pullsByTitle); + } + + return groundTruth(user); +} + +/** + * The observable facts the smoke run checks, read back from live Gitea (not from + * the plan) so the assertion compares the real repository against the declared + * ground truth rather than the plan against itself. + */ +export interface SeedSummary { + labelNames: string[]; + openIssueTitles: string[]; + closedIssueTitles: string[]; + selfAssignedIssueCount: number; + issuesWithCommentsCount: number; + pullTitles: string[]; + labeledPullTitles: string[]; + reviewedPullTitles: string[]; +} + +interface GiteaIssueSummary { + title: string; + state: string; + assignees: { login: string }[] | null; + comments: number; +} + +interface GiteaPullSummary { + number: number; + title: string; + labels: { name: string }[] | null; +} + +/** Read the live repository into the summary the smoke run asserts against. */ +export async function readSeedSummary( + access: BenchAccess, + coords: RepoCoords, +): Promise { + const repo = `/repos/${coords.owner}/${coords.repo}`; + const labels = await send(access, "GET", `${repo}/labels?limit=100`); + const issues = await send( + access, + "GET", + `${repo}/issues?type=issues&state=all&limit=100`, + ); + const pulls = await send(access, "GET", `${repo}/pulls?state=all&limit=100`); + + const reviewedPullTitles: string[] = []; + for (const pull of pulls) { + const reviews = await send<{ body: string }[]>( + access, + "GET", + `${repo}/pulls/${pull.number}/reviews`, + ); + if (reviews.length > 0) { + reviewedPullTitles.push(pull.title); + } + } + + return { + labelNames: labels.map((label) => label.name), + openIssueTitles: issues.filter((i) => i.state === "open").map((i) => i.title), + closedIssueTitles: issues.filter((i) => i.state === "closed").map((i) => i.title), + selfAssignedIssueCount: issues.filter((i) => (i.assignees ?? []).length > 0).length, + issuesWithCommentsCount: issues.filter((i) => i.comments > 0).length, + pullTitles: pulls.map((p) => p.title), + labeledPullTitles: pulls.filter((p) => (p.labels ?? []).length > 0).map((p) => p.title), + reviewedPullTitles, + }; +} diff --git a/package.json b/package.json index 15d7790..ef4e7e4 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,8 @@ "test:coverage": "vitest run --coverage", "test:e2e": "vitest run --config vitest.e2e.config.ts", "test:pack": "vitest run --config vitest.packaging.config.ts", - "test:bench": "vitest run --config vitest.bench.config.ts" + "test:bench": "vitest run --config vitest.bench.config.ts", + "test:bench:smoke": "vitest run --config vitest.bench-smoke.config.ts" }, "dependencies": { "@toon-format/toon": "^2.3.0", diff --git a/src/context.ts b/src/context.ts index 1fd9067..1049b78 100644 --- a/src/context.ts +++ b/src/context.ts @@ -96,7 +96,7 @@ function matchLoginsByHost(logins: TeaLogin[], host: string): TeaLogin[] { }); } -function selectLogin( +export function selectLogin( logins: TeaLogin[], loginName: string | undefined, remoteHost: string | undefined, diff --git a/tsconfig.json b/tsconfig.json index d1f9b9f..8f3ebb3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,5 +11,12 @@ "types": ["node"], "noEmit": true }, - "include": ["src", "test", "bench", "vitest.config.ts", "vitest.bench.config.ts"] + "include": [ + "src", + "test", + "bench", + "vitest.config.ts", + "vitest.bench.config.ts", + "vitest.bench-smoke.config.ts" + ] } diff --git a/vitest.bench-smoke.config.ts b/vitest.bench-smoke.config.ts new file mode 100644 index 0000000..22daeef --- /dev/null +++ b/vitest.bench-smoke.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + // The seed smoke tier: a single live end-to-end validation that provisioning + // and seeding a throwaway repository really brings it to the ground truth, + // and that re-seeding is idempotent. Like the e2e tier, it targets a live + // host — here the maintainer's own, discovered through gitea-axi's tea-login + // credential path — and skips cleanly when none is configured + // (GITEA_AXI_BENCH_LOGIN unset), which must count as a pass, not "no tests + // found". It is deliberately kept out of the deterministic bench tier. + include: ["bench/**/*.smoke.test.ts"], + testTimeout: 60_000, + hookTimeout: 180_000, + passWithNoTests: true, + }, +}); diff --git a/vitest.bench.config.ts b/vitest.bench.config.ts index 11abfeb..512bf92 100644 --- a/vitest.bench.config.ts +++ b/vitest.bench.config.ts @@ -8,6 +8,9 @@ export default defineConfig({ // source. It runs on its own via `test:bench` and is kept out of the main // fast tier so bench code never counts against src coverage thresholds. include: ["bench/**/*.test.ts"], + // The live seed smoke run is its own tier (vitest.bench-smoke.config.ts); it + // talks to a real Gitea host, so it stays out of this deterministic tier. + exclude: ["bench/**/*.smoke.test.ts", "**/node_modules/**"], passWithNoTests: true, }, }); -- 2.47.3