feat: add benchmark single-cell runner (task 0027)
Thread every benchmark layer to run one (arm, task, trial) cell end to end: provision and seed a throwaway repository, run the agent under the active arm bounded by a turn cap and a wall-clock backstop, audit the transcript, capture and score the post-run state, append the sample, and delete the repository. - runner.ts: runCell orchestration behind the BenchHost and AgentDriver seams, so the flow is unit-tested with fakes while the live wiring is validated by a smoke run; turn-cap and wall-clock failures are tagged confused-versus-hung, and a leaked transcript is flagged invalid. - audit.ts: the post-run transcript audit plus the shared foreignToolReason predicate both isolation enforcement points consume. - task.ts: the runnable BenchTask wrapper and one sample single-mutation task exercising the full path. - snapshot.ts: captureRepoState, the seed's read-back counterpart, in the RepoState shape the checker diffs against. - host.ts / sdk-driver.ts: the live BenchHost and the Claude Agent SDK driver (an optional peer, loaded via dynamic import) for real runs. - runner.smoke.test.ts: the live tracer-bullet tier, skipping cleanly when no host or SDK is configured.
This commit was merged in pull request #28.
This commit is contained in:
185
bench/snapshot.ts
Normal file
185
bench/snapshot.ts
Normal file
@@ -0,0 +1,185 @@
|
||||
// Post-run snapshot capture: read the entire scored surface of a live throwaway
|
||||
// repository back into the `RepoState` shape the checker diffs against. This is
|
||||
// the runner's counterpart to the seed — the seed writes the ground truth, this
|
||||
// reads the actual post-run state — so a mutation task can be scored by the
|
||||
// full-state diff (checker.ts) and any collateral change is caught.
|
||||
//
|
||||
// Like seed.ts, this is an imperative live boundary: its value is the real Gitea
|
||||
// API interaction, so it is exercised by the smoke run rather than mocked unit
|
||||
// tests. It reuses seed.ts's authenticated `send` helper so reads and writes share
|
||||
// one round-trip, and normalizes the few fields whose live representation differs
|
||||
// from the declared ground truth (notably label colours, which Gitea returns as
|
||||
// bare hex without the leading `#`).
|
||||
|
||||
import { send, type BenchAccess, type RepoCoords } from "./seed.js";
|
||||
import type {
|
||||
Comment,
|
||||
Issue,
|
||||
IssueState,
|
||||
Label,
|
||||
PullRequest,
|
||||
PullRequestState,
|
||||
RepoState,
|
||||
Review,
|
||||
ReviewKind,
|
||||
} from "./scoring-spec.js";
|
||||
|
||||
interface GiteaLabel {
|
||||
name: string;
|
||||
color: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
interface GiteaComment {
|
||||
body: string;
|
||||
user: { login: string };
|
||||
}
|
||||
|
||||
interface GiteaIssue {
|
||||
number: number;
|
||||
title: string;
|
||||
body: string;
|
||||
state: string;
|
||||
labels: { name: string }[] | null;
|
||||
assignees: { login: string }[] | null;
|
||||
}
|
||||
|
||||
interface GiteaPull {
|
||||
number: number;
|
||||
title: string;
|
||||
body: string;
|
||||
state: string;
|
||||
merged: boolean;
|
||||
labels: { name: string }[] | null;
|
||||
assignees: { login: string }[] | null;
|
||||
}
|
||||
|
||||
interface GiteaReview {
|
||||
user: { login: string };
|
||||
state: string;
|
||||
body: string;
|
||||
}
|
||||
|
||||
/** Colours compare against the declared ground truth's leading-`#` lowercase hex. */
|
||||
function normalizeColor(color: string): string {
|
||||
const bare = color.replace(/^#/, "").toLowerCase();
|
||||
return `#${bare}`;
|
||||
}
|
||||
|
||||
/** Gitea's applied-label and assignee arrays are nullable; reduce to name/login sets. */
|
||||
function labelNames(labels: { name: string }[] | null): string[] {
|
||||
return (labels ?? []).map((label) => label.name);
|
||||
}
|
||||
|
||||
function assigneeLogins(assignees: { login: string }[] | null): string[] {
|
||||
return (assignees ?? []).map((assignee) => assignee.login);
|
||||
}
|
||||
|
||||
/** Map a Gitea comment to the author/body pair the checker matches on. */
|
||||
function toComment(comment: GiteaComment): Comment {
|
||||
return { author: comment.user.login, body: comment.body };
|
||||
}
|
||||
|
||||
/** The Gitea review `state` verbs that map to a scored review kind; others are ignored. */
|
||||
const REVIEW_KIND: Record<string, ReviewKind> = {
|
||||
APPROVED: "approved",
|
||||
REQUEST_CHANGES: "request-changes",
|
||||
COMMENT: "comment",
|
||||
};
|
||||
|
||||
/** Read the comments (author and body) on an issue or pull request by number. */
|
||||
async function captureComments(
|
||||
access: BenchAccess,
|
||||
coords: RepoCoords,
|
||||
number: number,
|
||||
): Promise<Comment[]> {
|
||||
const comments = await send<GiteaComment[]>(
|
||||
access,
|
||||
"GET",
|
||||
`/repos/${coords.owner}/${coords.repo}/issues/${number}/comments`,
|
||||
);
|
||||
return comments.map(toComment);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a pull request's reviews, keeping only those whose Gitea state maps to a
|
||||
* scored review kind (a bare pending or review-request entry is dropped). Inline
|
||||
* review comments are left empty, matching the declared ground truth, which the
|
||||
* single-user seed never populates with inline comments.
|
||||
*/
|
||||
async function captureReviews(
|
||||
access: BenchAccess,
|
||||
coords: RepoCoords,
|
||||
number: number,
|
||||
): Promise<Review[]> {
|
||||
const reviews = await send<GiteaReview[]>(
|
||||
access,
|
||||
"GET",
|
||||
`/repos/${coords.owner}/${coords.repo}/pulls/${number}/reviews`,
|
||||
);
|
||||
const captured: Review[] = [];
|
||||
for (const review of reviews) {
|
||||
const kind = REVIEW_KIND[review.state];
|
||||
if (kind === undefined) {
|
||||
continue;
|
||||
}
|
||||
captured.push({ author: review.user.login, kind, body: review.body, comments: [] });
|
||||
}
|
||||
return captured;
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture the full post-run state of a throwaway repository — its labels, issues,
|
||||
* and pull requests with their applied labels, assignees, comments, and reviews —
|
||||
* in the shape the checker scores against. The volatile ids and timestamps the
|
||||
* ground truth drops are simply never read.
|
||||
*/
|
||||
export async function captureRepoState(
|
||||
access: BenchAccess,
|
||||
coords: RepoCoords,
|
||||
): Promise<RepoState> {
|
||||
const base = `/repos/${coords.owner}/${coords.repo}`;
|
||||
|
||||
const rawLabels = await send<GiteaLabel[]>(access, "GET", `${base}/labels?limit=100`);
|
||||
const labels: Label[] = rawLabels.map((label) => ({
|
||||
name: label.name,
|
||||
color: normalizeColor(label.color),
|
||||
...(label.description ? { description: label.description } : {}),
|
||||
}));
|
||||
|
||||
const rawIssues = await send<GiteaIssue[]>(
|
||||
access,
|
||||
"GET",
|
||||
`${base}/issues?type=issues&state=all&limit=100`,
|
||||
);
|
||||
const issues: Issue[] = [];
|
||||
for (const issue of rawIssues) {
|
||||
issues.push({
|
||||
number: issue.number,
|
||||
title: issue.title,
|
||||
body: issue.body,
|
||||
state: issue.state as IssueState,
|
||||
labels: labelNames(issue.labels),
|
||||
assignees: assigneeLogins(issue.assignees),
|
||||
comments: await captureComments(access, coords, issue.number),
|
||||
});
|
||||
}
|
||||
|
||||
const rawPulls = await send<GiteaPull[]>(access, "GET", `${base}/pulls?state=all&limit=100`);
|
||||
const pullRequests: PullRequest[] = [];
|
||||
for (const pull of rawPulls) {
|
||||
const state: PullRequestState = pull.merged ? "merged" : (pull.state as PullRequestState);
|
||||
pullRequests.push({
|
||||
number: pull.number,
|
||||
title: pull.title,
|
||||
body: pull.body,
|
||||
state,
|
||||
labels: labelNames(pull.labels),
|
||||
assignees: assigneeLogins(pull.assignees),
|
||||
comments: await captureComments(access, coords, pull.number),
|
||||
reviews: await captureReviews(access, coords, pull.number),
|
||||
});
|
||||
}
|
||||
|
||||
return { labels, issues, pullRequests };
|
||||
}
|
||||
Reference in New Issue
Block a user