.claude/tasks/0024-bench-checker-and-scoring-spec.md

@@ -12,8 +12,21 @@ This slice also defines the scoring-spec contract — a task's expected end stat
 
 ## Acceptance criteria
 
-- [ ] Given synthetic actual and expected state snapshots, the full-state diff passes when they match after normalization and fails when the actual state is missing the intended change.
-- [ ] The diff fails when the actual state carries collateral change beyond the intended mutation.
-- [ ] Normalization drops volatile identifiers and timestamps, matches comments by author and body, and compares label sets order-independently.
-- [ ] The read-task answer-match passes when the required facts are present in the final report and fails when a required fact is missing.
-- [ ] The scoring-spec contract expresses both a mutation's expected end state and a read's required answer facts.
+- [x] Given synthetic actual and expected state snapshots, the full-state diff passes when they match after normalization and fails when the actual state is missing the intended change.
+- [x] The diff fails when the actual state carries collateral change beyond the intended mutation.
+- [x] Normalization drops volatile identifiers and timestamps, matches comments by author and body, and compares label sets order-independently.
+- [x] The read-task answer-match passes when the required facts are present in the final report and fails when a required fact is missing.
+- [x] The scoring-spec contract expresses both a mutation's expected end state and a read's required answer facts.
+
+## Implementation Notes
+
+Two new files in `bench/`: `scoring-spec.ts` (the pure contract — `RepoState` and its `Label`/`Issue`/`PullRequest`/`Review`/`Comment` shapes, `RequiredFact`, and the `ScoringSpec` discriminated union) and `checker.ts` (the logic — `checkMutation`, `checkReadAnswer`, and a `score` entry point that dispatches on task kind). This mirrors the existing `result.ts` (shape) / `store.ts` (logic) split.
+
+Decisions and deviations worth flagging:
+
+- **Full-state diff covers the whole PR/issue surface, including reviews, assignees, and label definitions.** The criteria name only comments and label sets under normalization, but "diffing the entire post-run repository state" (User Story 5) and the contract's need to express the scored suite's review/merge/assignee tasks (criterion 5) make these part of the contract, not scope creep. They are groundwork the runner and task suite will populate.
+- **Reviews are matched order-independently**, consistent with how comments and labels are compared (spec line 9). A code review caught that reviews were initially order-dependent; this was fixed and covered by a test (`passes when a pull request's reviews match as a set despite differing order`). Review inline comments are likewise matched by author and body, order-independently.
+- **Failure diagnostics (`differences` naming the affected entity/label/comment/fact)** go beyond the bare pass/fail the criteria require, so a failed trial is traceable to what diverged (User Story 14 spirit). Heavily tested.
+- **Read-answer matching is deterministic substring matching** (case- and whitespace-normalized, with `anyOf` alternatives per fact), no LLM judge. This is intentionally naive — e.g. `"#42"` would match inside `"#420"` — and is mitigated by the task suite choosing disambiguating `anyOf` renderings rather than by the checker. The required facts carried in the `ScoringSpec` *are* the seeded ground truth for read tasks.
+- **`score` throws on a spec/submission kind mismatch** rather than silently scoring the wrong thing; this keeps the seam deterministic and is covered by a guard test.
+- No criteria were dropped; all five are satisfied.

bench/README.md

@@ -34,6 +34,8 @@ The raw component breakdown is retained on every sample so the data can be re-we
 - `result.ts` — the immutable result-record shape and its tags (arm, task, tier, trial, timestamp).
 - `store.ts` — the append-only, per-cell sample store that accumulates result records.
 - `guard.ts` — the authoritative tool-isolation guard plus the curated per-arm bin directory that backs it.
+- `scoring-spec.ts` — the scoring-spec contract: a task's expected end state (mutation) or required answer facts (read), consumed by the checker and produced by the runner and task suite.
+- `checker.ts` — the deterministic scorer: the full-state diff for mutation tasks and the answer-match for read tasks, plus the `score` entry point that dispatches on task kind.
 
 Later slices add the seed provisioning, the arm scaffolding, the single-cell runner, the task suite, the run-loop CLI, and the aggregator.
 

bench/checker.test.ts

@@ -0,0 +1,354 @@
+import { describe, expect, it } from "vitest";
+import { checkMutation, checkReadAnswer, score } from "./checker.js";
+import type { Submission } from "./checker.js";
+import type {
+  Issue,
+  Label,
+  PullRequest,
+  RepoState,
+  RequiredFact,
+  ScoringSpec,
+} from "./scoring-spec.js";
+
+/** Build an issue with sensible defaults, overridable per test. */
+function issue(overrides: Partial<Issue> = {}): Issue {
+  return {
+    number: 1,
+    title: "Login button misaligned",
+    body: "The submit button overflows on mobile.",
+    state: "open",
+    labels: [],
+    assignees: [],
+    comments: [],
+    ...overrides,
+  };
+}
+
+/** Build a label with sensible defaults, overridable per test. */
+function label(overrides: Partial<Label> = {}): Label {
+  return {
+    name: "bug",
+    color: "#d73a4a",
+    ...overrides,
+  };
+}
+
+/** Build a pull request with sensible defaults, overridable per test. */
+function pr(overrides: Partial<PullRequest> = {}): PullRequest {
+  return {
+    number: 5,
+    title: "Fix login button alignment",
+    body: "Closes #1.",
+    state: "merged",
+    labels: [],
+    assignees: [],
+    comments: [],
+    reviews: [],
+    ...overrides,
+  };
+}
+
+/** Build a full repository snapshot with sensible defaults, overridable per test. */
+function repo(overrides: Partial<RepoState> = {}): RepoState {
+  return {
+    labels: [],
+    issues: [],
+    pullRequests: [],
+    ...overrides,
+  };
+}
+
+describe("checkMutation", () => {
+  it("passes when the actual state matches the expected end state after normalization", () => {
+    // Intended change: issue #1 is closed and the "bug" label applied. The expected
+    // end state fixes that outcome.
+    const expected = repo({
+      labels: [label({ name: "bug", color: "#d73a4a" })],
+      issues: [issue({ number: 1, state: "closed", labels: ["bug"] })],
+    });
+
+    // The actual post-run snapshot embodies the same outcome, but carries volatile
+    // host-assigned ids/timestamps and lists the label set in a different order —
+    // all of which normalization must ignore.
+    const actual = repo({
+      labels: [label({ name: "bug", color: "#d73a4a" })],
+      issues: [
+        issue({
+          number: 1,
+          state: "closed",
+          labels: ["bug"],
+          id: 4201,
+          createdAt: "2026-07-15T09:00:00Z",
+          updatedAt: "2026-07-16T10:30:00Z",
+        }),
+      ],
+    });
+
+    expect(checkMutation(expected, actual)).toEqual({ pass: true });
+  });
+
+  it("fails and names the affected entity when the actual state is missing the intended change", () => {
+    // Intended change: issue #1 is closed. The expected end state fixes that outcome.
+    const expected = repo({
+      issues: [issue({ number: 1, state: "closed" })],
+    });
+
+    // The change did not happen: issue #1 is still open in the actual post-run state.
+    const actual = repo({
+      issues: [issue({ number: 1, state: "open" })],
+    });
+
+    const result = checkMutation(expected, actual);
+
+    expect(result.pass).toBe(false);
+    // A diagnosable failure names the affected entity so a failed trial can be
+    // traced back to what diverged — here, issue #1.
+    if (result.pass === false) {
+      expect(result.differences.some((d) => d.includes("#1"))).toBe(true);
+    }
+  });
+
+  it("fails and names the stray label when the actual state carries collateral change", () => {
+    // Intended change: issue #1 closed, carrying no applied labels. Both snapshots
+    // define the same repository labels and agree on the closed state of #1.
+    const expected = repo({
+      labels: [
+        label({ name: "bug", color: "#d73a4a" }),
+        label({ name: "wontfix", color: "#ffffff" }),
+      ],
+      issues: [issue({ number: 1, state: "closed", labels: [] })],
+    });
+
+    // Collateral damage: an extra "wontfix" label was applied to issue #1 even
+    // though the expected end state leaves it unlabelled.
+    const actual = repo({
+      labels: [
+        label({ name: "bug", color: "#d73a4a" }),
+        label({ name: "wontfix", color: "#ffffff" }),
+      ],
+      issues: [issue({ number: 1, state: "closed", labels: ["wontfix"] })],
+    });
+
+    const result = checkMutation(expected, actual);
+
+    expect(result.pass).toBe(false);
+    // The stray label must be named so the collateral change is diagnosable —
+    // knowing #1's labels merely "differ" does not say what was wrongly applied.
+    if (result.pass === false) {
+      expect(result.differences.some((d) => d.includes("wontfix"))).toBe(true);
+    }
+  });
+
+  it("passes when comments match by author and body despite differing order and volatile ids", () => {
+    // The expected issue carries two comments in one order, with one set of
+    // host-assigned ids and timestamps.
+    const expected = repo({
+      issues: [
+        issue({
+          number: 1,
+          state: "closed",
+          comments: [
+            { author: "octocat", body: "Reproduced on mobile Safari.", id: 11, createdAt: "2026-07-15T09:00:00Z" },
+            { author: "maintainer", body: "Fixed in the latest build.", id: 12, createdAt: "2026-07-15T10:00:00Z" },
+          ],
+        }),
+      ],
+    });
+
+    // The actual issue carries the same set of comments by (author, body), but in
+    // the reverse order and with entirely different volatile ids and timestamps —
+    // all of which normalization must ignore.
+    const actual = repo({
+      issues: [
+        issue({
+          number: 1,
+          state: "closed",
+          comments: [
+            { author: "maintainer", body: "Fixed in the latest build.", id: 907, createdAt: "2026-07-16T14:22:00Z" },
+            { author: "octocat", body: "Reproduced on mobile Safari.", id: 906, createdAt: "2026-07-16T14:20:00Z" },
+          ],
+        }),
+      ],
+    });
+
+    expect(checkMutation(expected, actual)).toEqual({ pass: true });
+  });
+
+  it("fails and names the missing comment when the actual state lacks a required comment", () => {
+    // Intended change: issue #1 must carry a specific maintainer comment asking
+    // for a reproduction case.
+    const expected = repo({
+      issues: [
+        issue({
+          number: 1,
+          comments: [{ author: "maintainer", body: "Please add a reproduction case." }],
+        }),
+      ],
+    });
+
+    // The agent never posted that comment: the actual issue has no comments.
+    const actual = repo({
+      issues: [issue({ number: 1, comments: [] })],
+    });
+
+    const result = checkMutation(expected, actual);
+
+    expect(result.pass).toBe(false);
+    // The specific missing comment must be identifiable from its own text so the
+    // divergence is diagnosable, not just "comments differ".
+    if (result.pass === false) {
+      expect(result.differences.some((d) => d.includes("Please add a reproduction case."))).toBe(
+        true,
+      );
+    }
+  });
+
+  it("passes when an issue's applied labels match as a set despite differing order", () => {
+    // The expected end state applies two labels to issue #1 in one order.
+    const expected = repo({
+      labels: [
+        label({ name: "bug", color: "#d73a4a" }),
+        label({ name: "priority:high", color: "#b60205" }),
+      ],
+      issues: [issue({ number: 1, state: "closed", labels: ["bug", "priority:high"] })],
+    });
+
+    // The actual issue carries the same applied labels, but lists them in the
+    // reverse order — which an order-independent set comparison must ignore.
+    const actual = repo({
+      labels: [
+        label({ name: "bug", color: "#d73a4a" }),
+        label({ name: "priority:high", color: "#b60205" }),
+      ],
+      issues: [issue({ number: 1, state: "closed", labels: ["priority:high", "bug"] })],
+    });
+
+    expect(checkMutation(expected, actual)).toEqual({ pass: true });
+  });
+
+  it("passes when a pull request's reviews match as a set despite differing order", () => {
+    // The expected pull request #5 carries two reviews in one order.
+    const expected = repo({
+      pullRequests: [
+        pr({
+          number: 5,
+          state: "merged",
+          reviews: [
+            { author: "alexion", kind: "comment", body: "first pass", comments: [] },
+            { author: "alexion", kind: "approved", body: "looks good", comments: [] },
+          ],
+        }),
+      ],
+    });
+
+    // The actual pull request carries the same two reviews (each identified by
+    // author, kind, and body) but lists them in the reverse order — which an
+    // order-independent set comparison must ignore, as it already does for
+    // comments and labels.
+    const actual = repo({
+      pullRequests: [
+        pr({
+          number: 5,
+          state: "merged",
+          reviews: [
+            { author: "alexion", kind: "approved", body: "looks good", comments: [] },
+            { author: "alexion", kind: "comment", body: "first pass", comments: [] },
+          ],
+        }),
+      ],
+    });
+
+    expect(checkMutation(expected, actual)).toEqual({ pass: true });
+  });
+});
+
+describe("checkReadAnswer", () => {
+  it("passes when every required fact is present in the final report", () => {
+    // A read task requires the agent to report a count of open issues and a
+    // specific issue number; each fact lists acceptable renderings.
+    const facts: RequiredFact[] = [
+      { description: "count of open issues", anyOf: ["3 open issues", "three open issues"] },
+      { description: "the stale issue's number", anyOf: ["#42", "issue 42"] },
+    ];
+
+    // The report plainly contains a rendering of each fact.
+    const report = "I found 3 open issues; the oldest untouched one is #42.";
+
+    expect(checkReadAnswer(facts, report)).toEqual({ pass: true });
+  });
+
+  it("fails and names the missing fact when a required fact is absent from the report", () => {
+    // A read task requires two facts, each named by a distinctive description.
+    const facts: RequiredFact[] = [
+      { description: "the count of open bug issues", anyOf: ["2 open bug issues", "two open bug issues"] },
+      { description: "the newest issue number", anyOf: ["#57", "issue 57"] },
+    ];
+
+    // The report renders the first fact but omits any rendering of the second.
+    const report = "There are 2 open bug issues in the repository.";
+
+    const result = checkReadAnswer(facts, report);
+
+    expect(result.pass).toBe(false);
+    // The unmet fact must be identifiable by its own description so a failed read
+    // task can be diagnosed — not just "a required fact is missing".
+    if (result.pass === false) {
+      expect(result.differences.some((d) => d.includes("the newest issue number"))).toBe(true);
+    }
+  });
+
+  it("passes on an alternate anyOf rendering that differs in case and whitespace", () => {
+    // The fact offers two acceptable renderings of the same count.
+    const facts: RequiredFact[] = [
+      { description: "count of open issues", anyOf: ["3 open issues", "three open issues"] },
+    ];
+
+    // The report contains only the SECOND rendering, in different case and with
+    // irregular internal whitespace — which case- and whitespace-insensitive
+    // matching must still accept.
+    const report = "There are   THREE   Open Issues left.";
+
+    expect(checkReadAnswer(facts, report)).toEqual({ pass: true });
+  });
+});
+
+describe("score", () => {
+  it("dispatches on the spec kind: full-state diff for mutations, answer-match for reads", () => {
+    // A mutation spec is scored by diffing the submitted repository state against
+    // the expected end state; a matching submission passes.
+    const expectedState = repo({
+      issues: [issue({ number: 1, state: "closed", labels: ["bug"] })],
+      labels: [label({ name: "bug", color: "#d73a4a" })],
+    });
+    const mutationSpec: ScoringSpec = { kind: "mutation", expected: expectedState };
+    const matchingSubmission: Submission = { kind: "mutation", state: expectedState };
+
+    expect(score(mutationSpec, matchingSubmission)).toEqual({ pass: true });
+
+    // A read spec is scored by matching required facts in the submitted report.
+    const readSpec: ScoringSpec = {
+      kind: "read",
+      facts: [{ description: "the answer", anyOf: ["42"] }],
+    };
+
+    const passingRead: Submission = { kind: "read", report: "the answer is 42" };
+    expect(score(readSpec, passingRead)).toEqual({ pass: true });
+
+    // And a read submission lacking the required fact fails.
+    const failingRead: Submission = { kind: "read", report: "no idea, sorry" };
+    expect(score(readSpec, failingRead).pass).toBe(false);
+  });
+
+  it("throws when the submission's kind does not match the spec's kind", () => {
+    // A mutation spec paired with a read submission is a caller error, not a
+    // scoreable outcome: score must reject it rather than silently score the
+    // wrong thing.
+    const mutationSpec: ScoringSpec = {
+      kind: "mutation",
+      expected: repo({ issues: [issue({ number: 1, state: "closed" })] }),
+    };
+    const readSubmission: Submission = { kind: "read", report: "" };
+
+    expect(() => score(mutationSpec, readSubmission)).toThrow();
+  });
+});

bench/checker.ts

@@ -0,0 +1,283 @@
+// The checker: the pure scoring seam that turns a completed run into a
+// deterministic pass/fail. A mutation task is scored by diffing the entire
+// post-run repository state against the expected end state (so both the intended
+// change and any collateral damage are caught); a read task is scored by matching
+// its required answer facts against the agent's final report, with no LLM judge.
+//
+// The checker is fed synthetic state snapshots and expected states; capturing the
+// live state from a real repository is the runner's job. The scoring-spec contract
+// it consumes lives in scoring-spec.ts.
+
+import type {
+  Comment,
+  Label,
+  PullRequest,
+  RepoState,
+  RequiredFact,
+  Review,
+  ScoringSpec,
+} from "./scoring-spec.js";
+
+/**
+ * The checker's verdict. On failure it carries the human-readable differences —
+ * each missing intended change or collateral change for a mutation, or each
+ * missing fact for a read — so a failed trial can be diagnosed from the record.
+ */
+export type CheckResult = { pass: true } | { pass: false; differences: string[] };
+
+/** What a completed run submits for scoring, tagged by the kind of task it was. */
+export type Submission =
+  | { kind: "mutation"; state: RepoState }
+  | { kind: "read"; report: string };
+
+/**
+ * Score a mutation task by diffing the full actual state against the expected end
+ * state. The diff walks the whole snapshot, so a difference is raised whether the
+ * actual state is missing the intended change or carries a collateral one. The
+ * volatile, host-assigned ids and timestamps are simply never compared, so two
+ * states that agree on the meaningful fields match regardless of them.
+ */
+export function checkMutation(expected: RepoState, actual: RepoState): CheckResult {
+  const differences: string[] = [];
+  diffLabels(expected.labels, actual.labels, differences);
+  diffByNumber("issue", expected.issues, actual.issues, differences, diffConversation);
+  diffByNumber("pull request", expected.pullRequests, actual.pullRequests, differences, diffPullRequest);
+  return differences.length === 0 ? { pass: true } : { pass: false, differences };
+}
+
+/** Diff the repository's label definitions, matched by name. */
+function diffLabels(expected: Label[], actual: Label[], differences: string[]): void {
+  const expectedByName = new Map(expected.map((l) => [l.name, l]));
+  const actualByName = new Map(actual.map((l) => [l.name, l]));
+  for (const [name, e] of expectedByName) {
+    const a = actualByName.get(name);
+    if (a === undefined) {
+      differences.push(`missing label "${name}"`);
+      continue;
+    }
+    if (e.color !== a.color) {
+      differences.push(`label "${name}" color expected "${e.color}" but was "${a.color}"`);
+    }
+    if ((e.description ?? "") !== (a.description ?? "")) {
+      differences.push(`label "${name}" description differs`);
+    }
+  }
+  for (const name of actualByName.keys()) {
+    if (!expectedByName.has(name)) {
+      differences.push(`unexpected label "${name}" (collateral change)`);
+    }
+  }
+}
+
+/**
+ * Match two lists of numbered entities (issues or pull requests) by their stable
+ * number, reporting any expected entity missing from the actual state and any
+ * actual entity the expected state does not contain (collateral), then comparing
+ * the fields of each matched pair.
+ */
+function diffByNumber<T extends { number: number }>(
+  kind: string,
+  expected: T[],
+  actual: T[],
+  differences: string[],
+  compareFields: (where: string, e: T, a: T, differences: string[]) => void,
+): void {
+  const expectedByNumber = new Map(expected.map((e) => [e.number, e]));
+  const actualByNumber = new Map(actual.map((a) => [a.number, a]));
+  for (const [number, e] of expectedByNumber) {
+    const a = actualByNumber.get(number);
+    if (a === undefined) {
+      differences.push(`missing ${kind} #${number}`);
+      continue;
+    }
+    compareFields(`${kind} #${number}`, e, a, differences);
+  }
+  for (const number of actualByNumber.keys()) {
+    if (!expectedByNumber.has(number)) {
+      differences.push(`unexpected ${kind} #${number} (collateral change)`);
+    }
+  }
+}
+
+/**
+ * The conversation surface an issue and a pull request share: title, body, state,
+ * applied labels, assignees, and comments. State is compared as an opaque string
+ * so an issue's open/closed and a pull request's open/closed/merged both flow
+ * through the same diff.
+ */
+interface Conversation {
+  title: string;
+  body: string;
+  state: string;
+  labels: string[];
+  assignees: string[];
+  comments: Comment[];
+}
+
+/** Diff the conversation fields common to issues and pull requests. */
+function diffConversation(where: string, e: Conversation, a: Conversation, differences: string[]): void {
+  diffScalar(where, "title", e.title, a.title, differences);
+  diffScalar(where, "body", e.body, a.body, differences);
+  diffScalar(where, "state", e.state, a.state, differences);
+  diffSet(where, "label", e.labels, a.labels, differences);
+  diffSet(where, "assignee", e.assignees, a.assignees, differences);
+  diffComments(where, e.comments, a.comments, differences);
+}
+
+/** Diff a pull request: its shared conversation surface plus its reviews. */
+function diffPullRequest(where: string, e: PullRequest, a: PullRequest, differences: string[]): void {
+  diffConversation(where, e, a, differences);
+  diffReviews(where, e.reviews, a.reviews, differences);
+}
+
+function diffScalar(where: string, field: string, e: string, a: string, differences: string[]): void {
+  if (e !== a) {
+    differences.push(`${where} ${field} expected "${e}" but was "${a}"`);
+  }
+}
+
+/**
+ * Diff two order-independent sets of named things (applied labels, assignees),
+ * naming each element the expected state requires but the actual state lacks, and
+ * each the actual state carries but the expected state does not (collateral), so
+ * the divergence is diagnosable down to the specific label or assignee.
+ */
+function diffSet(where: string, noun: string, e: string[], a: string[], differences: string[]): void {
+  const { missing, extra } = matchByKey(a, e, (name) => name);
+  for (const name of missing) {
+    differences.push(`${where} missing ${noun} "${name}"`);
+  }
+  for (const name of extra) {
+    differences.push(`${where} unexpected ${noun} "${name}" (collateral change)`);
+  }
+}
+
+/**
+ * Diff two sets of comments, matched by author and body (volatile id and
+ * timestamp ignored) and compared order-independently. Each comment the expected
+ * state requires but the actual lacks, and each the actual carries but the
+ * expected does not (collateral), is named by its author and body.
+ */
+function diffComments(where: string, e: Comment[], a: Comment[], differences: string[]): void {
+  const { missing, extra } = matchByKey(a, e, commentKey);
+  for (const comment of missing) {
+    differences.push(`${where} missing comment ${describeComment(comment)}`);
+  }
+  for (const comment of extra) {
+    differences.push(`${where} unexpected comment ${describeComment(comment)} (collateral change)`);
+  }
+}
+
+/** The key a comment is matched by: its author and body, volatile fields dropped. */
+function commentKey(comment: Comment): string {
+  return JSON.stringify([comment.author, comment.body]);
+}
+
+/** A readable rendering of the author and body a comment is matched by. */
+function describeComment(comment: Comment): string {
+  return `from ${comment.author}: ${JSON.stringify(comment.body)}`;
+}
+
+/**
+ * Diff two sets of reviews, matched order-independently — like comments and
+ * labels — by author, kind, body, and their inline comments (themselves matched
+ * by author and body, order-independently). Each review the expected state
+ * requires but the actual lacks, and each collateral review the actual carries,
+ * is named by its author, kind, and body.
+ */
+function diffReviews(where: string, e: Review[], a: Review[], differences: string[]): void {
+  const { missing, extra } = matchByKey(a, e, reviewKey);
+  for (const review of missing) {
+    differences.push(`${where} missing review ${describeReview(review)}`);
+  }
+  for (const review of extra) {
+    differences.push(`${where} unexpected review ${describeReview(review)} (collateral change)`);
+  }
+}
+
+/** The key a review is matched by: author, kind, body, and its inline comments as a set. */
+function reviewKey(review: Review): string {
+  const comments = review.comments.map((c) => [c.author, c.body]).sort();
+  return JSON.stringify([review.author, review.kind, review.body, comments]);
+}
+
+/** A readable rendering of the author, kind, and body a review is matched by. */
+function describeReview(review: Review): string {
+  return `by ${review.author} (${review.kind}): ${JSON.stringify(review.body)}`;
+}
+
+/**
+ * Match two collections order-independently by a key, returning the expected
+ * items with no actual counterpart (`missing`) and the actual items with no
+ * expected counterpart (`extra`). Each expected item matches at most one actual
+ * item, so duplicates are honored (two identical expected comments require two
+ * in the actual state).
+ */
+function matchByKey<T>(
+  actual: T[],
+  expected: T[],
+  key: (item: T) => string,
+): { missing: T[]; extra: T[] } {
+  const unmatched = expected.map((item) => ({ item, key: key(item) }));
+  const extra: T[] = [];
+  for (const item of actual) {
+    const k = key(item);
+    const index = unmatched.findIndex((candidate) => candidate.key === k);
+    if (index >= 0) {
+      unmatched.splice(index, 1);
+    } else {
+      extra.push(item);
+    }
+  }
+  return { missing: unmatched.map((entry) => entry.item), extra };
+}
+
+/**
+ * Score a read task by matching its required answer facts against the agent's
+ * final report — no LLM judge. A fact is present when the report contains any one
+ * of its acceptable renderings, compared after lower-casing and collapsing
+ * whitespace so trivial phrasing differences do not matter. The answer passes
+ * only when every required fact is present.
+ */
+export function checkReadAnswer(facts: RequiredFact[], report: string): CheckResult {
+  const haystack = normalizeText(report);
+  const missing = facts.filter(
+    (fact) => !fact.anyOf.some((rendering) => haystack.includes(normalizeText(rendering))),
+  );
+  if (missing.length === 0) {
+    return { pass: true };
+  }
+  return {
+    pass: false,
+    differences: missing.map((fact) => `missing required fact: ${fact.description}`),
+  };
+}
+
+/** Lower-case and collapse runs of whitespace so incidental phrasing does not matter. */
+function normalizeText(text: string): string {
+  return text.toLowerCase().replace(/\s+/g, " ").trim();
+}
+
+/**
+ * Score a completed run against its scoring spec, dispatching on the task kind: a
+ * mutation spec is scored by the full-state diff against the submitted repository
+ * state, a read spec by the answer-match against the submitted report. The
+ * submission's kind must match the spec's — a mismatch is a caller error and
+ * throws, rather than silently scoring the wrong thing.
+ */
+export function score(spec: ScoringSpec, submission: Submission): CheckResult {
+  if (spec.kind === "mutation") {
+    if (submission.kind !== "mutation") {
+      throw new Error(
+        `a mutation spec must be scored against a mutation submission, got "${submission.kind}"`,
+      );
+    }
+    return checkMutation(spec.expected, submission.state);
+  }
+  if (submission.kind !== "read") {
+    throw new Error(
+      `a read spec must be scored against a read submission, got "${submission.kind}"`,
+    );
+  }
+  return checkReadAnswer(spec.facts, submission.report);
+}

bench/scoring-spec.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, it } from "vitest";
+import { score } from "./checker.js";
+import type { ScoringSpec } from "./scoring-spec.js";
+
+describe("ScoringSpec contract", () => {
+  it("expresses both a mutation's expected end state and a read's required answer facts", () => {
+    // A mutation spec fixes a rich expected end state: a labelled, commented,
+    // closed issue AND a merged pull request carrying a review. This proves the
+    // contract can express the whole scored surface, not just a single field.
+    const mutationSpec: ScoringSpec = {
+      kind: "mutation",
+      expected: {
+        labels: [{ name: "bug", color: "#d73a4a", description: "Something is broken" }],
+        issues: [
+          {
+            number: 1,
+            title: "Login button misaligned",
+            body: "Overflows on mobile.",
+            state: "closed",
+            labels: ["bug"],
+            assignees: ["octocat"],
+            comments: [{ author: "maintainer", body: "Fixed in the latest build." }],
+          },
+        ],
+        pullRequests: [
+          {
+            number: 2,
+            title: "Fix login button alignment",
+            body: "Closes #1.",
+            state: "merged",
+            labels: ["bug"],
+            assignees: ["octocat"],
+            comments: [{ author: "octocat", body: "Ready for review." }],
+            reviews: [
+              {
+                author: "maintainer",
+                kind: "approved",
+                body: "Looks good.",
+                comments: [{ author: "maintainer", body: "Nice fix." }],
+              },
+            ],
+          },
+        ],
+      },
+    };
+
+    expect(mutationSpec.kind).toBe("mutation");
+    // Scoring the very state the spec fixes must pass — the contract round-trips
+    // through the scorer.
+    expect(score(mutationSpec, { kind: "mutation", state: mutationSpec.expected })).toEqual({
+      pass: true,
+    });
+
+    // A read spec instead carries required answer facts. This proves the contract
+    // can express the read-task family.
+    const readSpec: ScoringSpec = {
+      kind: "read",
+      facts: [
+        { description: "count of open issues", anyOf: ["3 open issues", "three open issues"] },
+        { description: "the stale issue's number", anyOf: ["#42", "issue 42"] },
+      ],
+    };
+
+    expect(readSpec.kind).toBe("read");
+    // A report rendering both facts must pass.
+    const report = "There are 3 open issues; the oldest untouched one is #42.";
+    expect(score(readSpec, { kind: "read", report })).toEqual({ pass: true });
+  });
+});

bench/scoring-spec.ts

@@ -0,0 +1,147 @@
+// The scoring-spec contract: a task's expected outcome, in the form the checker
+// consumes and the runner and task suite produce. Two kinds mirror the two ways
+// the benchmark scores a run — a mutation task fixes the repository's expected
+// end state (diffed in full so collateral damage is caught), and a read task
+// fixes the facts the agent's final report must contain (matched deterministically,
+// with no LLM judge).
+//
+// These are pure contract types with no logic; the checker (checker.ts) is the
+// seam that scores an actual run against a spec of either kind. The benchmark's
+// own vocabulary (arm, cell, checker, seed) is documented in bench/README.md and
+// the benchmark-harness spec, deliberately kept out of the tool's own domain
+// glossary.
+
+/**
+ * A repository label definition. The seed fixes each label's colour, so colour
+ * and description are part of the expected end state; applied label *names* on an
+ * issue or pull request are compared separately, as an order-independent set.
+ */
+export interface Label {
+  name: string;
+  color: string;
+  description?: string;
+}
+
+/**
+ * One comment on an issue, pull request, or review. Comments are matched by
+ * author and body; the host-assigned id and timestamps are volatile and are
+ * dropped before comparison.
+ */
+export interface Comment {
+  author: string;
+  body: string;
+  /** Volatile: host-assigned, dropped by normalization. */
+  id?: number;
+  /** Volatile: dropped by normalization. */
+  createdAt?: string;
+}
+
+/** An issue's open/closed state. */
+export type IssueState = "open" | "closed";
+
+/** A pull request's state; unlike an issue, it may also be merged. */
+export type PullRequestState = "open" | "closed" | "merged";
+
+/** The kind of review a single user may leave on a pull request. */
+export type ReviewKind = "comment" | "approved" | "request-changes";
+
+/**
+ * One review on a pull request. The single-user seed allows comment-type reviews
+ * (and, where the host permits self-review, approvals and change requests).
+ * Reviews are matched by author, kind, body, and their inline comments; the
+ * host-assigned id and timestamp are volatile.
+ */
+export interface Review {
+  author: string;
+  kind: ReviewKind;
+  body: string;
+  /** Inline review comments; matched by author and body. */
+  comments: Comment[];
+  /** Volatile: host-assigned, dropped by normalization. */
+  id?: number;
+  /** Volatile: dropped by normalization. */
+  createdAt?: string;
+}
+
+/**
+ * One issue in the expected (or actual) repository state. The issue number is
+ * deterministic ground truth from the seed and keys the diff; the host-assigned
+ * id and timestamps are volatile and dropped.
+ */
+export interface Issue {
+  number: number;
+  title: string;
+  body: string;
+  state: IssueState;
+  /** Applied label names; compared as an order-independent set. */
+  labels: string[];
+  /** Assignee usernames; compared as an order-independent set. */
+  assignees: string[];
+  /** Comments; matched by author and body. */
+  comments: Comment[];
+  /** Volatile: host-assigned, dropped by normalization. */
+  id?: number;
+  /** Volatile: dropped by normalization. */
+  createdAt?: string;
+  /** Volatile: dropped by normalization. */
+  updatedAt?: string;
+}
+
+/**
+ * One pull request in the expected (or actual) repository state. Shares the
+ * conversation surface with an issue (labels, assignees, comments) and adds the
+ * merged state and reviews.
+ */
+export interface PullRequest {
+  number: number;
+  title: string;
+  body: string;
+  state: PullRequestState;
+  /** Applied label names; compared as an order-independent set. */
+  labels: string[];
+  /** Assignee usernames; compared as an order-independent set. */
+  assignees: string[];
+  /** Comments; matched by author and body. */
+  comments: Comment[];
+  /** Reviews; matched by author, kind, body, and inline comments. */
+  reviews: Review[];
+  /** Volatile: host-assigned, dropped by normalization. */
+  id?: number;
+  /** Volatile: dropped by normalization. */
+  createdAt?: string;
+  /** Volatile: dropped by normalization. */
+  updatedAt?: string;
+}
+
+/**
+ * A full snapshot of the throwaway repository's scored surface. A mutation task's
+ * expected end state is one of these; the checker captures the actual post-run
+ * state in the same shape and diffs the two in full, so both the intended change
+ * and any collateral damage are caught.
+ */
+export interface RepoState {
+  labels: Label[];
+  issues: Issue[];
+  pullRequests: PullRequest[];
+}
+
+/**
+ * One fact a read task's answer must contain. The fact is satisfied when the
+ * agent's final report contains any one of `anyOf`'s renderings (after
+ * whitespace and case normalization), so a count or a name can be phrased
+ * variously without resorting to an LLM judge. `description` names the fact in
+ * diagnostics when it is missing.
+ */
+export interface RequiredFact {
+  description: string;
+  anyOf: string[];
+}
+
+/**
+ * A task's scoring spec: either a mutation's expected end state or a read's
+ * required answer facts. The runner and task suite produce one of these per
+ * task; the checker consumes it to turn a completed run into a pass/fail.
+ */
+export type ScoringSpec =
+  | { kind: "mutation"; expected: RepoState }
+  | { kind: "read"; facts: RequiredFact[] };