// The scoring-spec contract: a task's expected outcome, in the form the checker // consumes and the runner and task suite produce. Two kinds mirror the two ways // the benchmark scores a run — a mutation task fixes the repository's expected // end state (diffed in full so collateral damage is caught), and a read task // fixes the facts the agent's final report must contain (matched deterministically, // with no LLM judge). // // These are pure contract types with no logic; the checker (checker.ts) is the // seam that scores an actual run against a spec of either kind. The benchmark's // own vocabulary (arm, cell, checker, seed) is documented in bench/README.md and // the benchmark-harness spec, deliberately kept out of the tool's own domain // glossary. /** * A repository label definition. The seed fixes each label's colour, so colour * and description are part of the expected end state; applied label *names* on an * issue or pull request are compared separately, as an order-independent set. */ export interface Label { name: string; color: string; description?: string; } /** * One comment on an issue, pull request, or review. Comments are matched by * author and body; the host-assigned id and timestamps are volatile and are * dropped before comparison. */ export interface Comment { author: string; body: string; /** Volatile: host-assigned, dropped by normalization. */ id?: number; /** Volatile: dropped by normalization. */ createdAt?: string; } /** An issue's open/closed state. */ export type IssueState = "open" | "closed"; /** A pull request's state; unlike an issue, it may also be merged. */ export type PullRequestState = "open" | "closed" | "merged"; /** The kind of review a single user may leave on a pull request. */ export type ReviewKind = "comment" | "approved" | "request-changes"; /** * One review on a pull request. The single-user seed allows comment-type reviews * (and, where the host permits self-review, approvals and change requests). * Reviews are matched by author, kind, body, and their inline comments; the * host-assigned id and timestamp are volatile. */ export interface Review { author: string; kind: ReviewKind; body: string; /** Inline review comments; matched by author and body. */ comments: Comment[]; /** Volatile: host-assigned, dropped by normalization. */ id?: number; /** Volatile: dropped by normalization. */ createdAt?: string; } /** * One issue in the expected (or actual) repository state. The issue number is * deterministic ground truth from the seed and keys the diff; the host-assigned * id and timestamps are volatile and dropped. */ export interface Issue { number: number; title: string; body: string; state: IssueState; /** Applied label names; compared as an order-independent set. */ labels: string[]; /** Assignee usernames; compared as an order-independent set. */ assignees: string[]; /** Comments; matched by author and body. */ comments: Comment[]; /** Volatile: host-assigned, dropped by normalization. */ id?: number; /** Volatile: dropped by normalization. */ createdAt?: string; /** Volatile: dropped by normalization. */ updatedAt?: string; } /** * One pull request in the expected (or actual) repository state. Shares the * conversation surface with an issue (labels, assignees, comments) and adds the * merged state and reviews. */ export interface PullRequest { number: number; title: string; body: string; state: PullRequestState; /** Applied label names; compared as an order-independent set. */ labels: string[]; /** Assignee usernames; compared as an order-independent set. */ assignees: string[]; /** Comments; matched by author and body. */ comments: Comment[]; /** Reviews; matched by author, kind, body, and inline comments. */ reviews: Review[]; /** Volatile: host-assigned, dropped by normalization. */ id?: number; /** Volatile: dropped by normalization. */ createdAt?: string; /** Volatile: dropped by normalization. */ updatedAt?: string; } /** * A full snapshot of the throwaway repository's scored surface. A mutation task's * expected end state is one of these; the checker captures the actual post-run * state in the same shape and diffs the two in full, so both the intended change * and any collateral damage are caught. */ export interface RepoState { labels: Label[]; issues: Issue[]; pullRequests: PullRequest[]; } /** * One fact a read task's answer must contain. The fact is satisfied when the * agent's final report contains any one of `anyOf`'s renderings (after * whitespace and case normalization), so a count or a name can be phrased * variously without resorting to an LLM judge. `description` names the fact in * diagnostics when it is missing. */ export interface RequiredFact { description: string; anyOf: string[]; } /** * A task's scoring spec: either a mutation's expected end state or a read's * required answer facts. The runner and task suite produce one of these per * task; the checker consumes it to turn a completed run into a pass/fail. */ export type ScoringSpec = | { kind: "mutation"; expected: RepoState } | { kind: "read"; facts: RequiredFact[] };