chore(vendor): import pi 1.3.0

Imported as a linear vendor commit so forge rebases do not replay the upstream root tree.

git-subtree-dir: third_party/pi

git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0

third_party/pi/packages/evals/test/acme-server.test.ts

@@ -0,0 +1,128 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
+import {
+	type AcmeServer,
+	createAcmeServer,
+	OPENAI_MODEL_ID,
+	OPENAI_PROBE_PROMPT,
+	OPENAI_PROBE_RESPONSE,
+	STREAM_MODEL_ID,
+	STREAM_PROBE_PROMPT,
+	STREAM_PROBE_RESPONSE,
+} from "../evals/acme-server.ts";
+
+type Fixture = {
+	mode: "openai" | "stream";
+	url: (server: AcmeServer) => string;
+	headers: Record<string, string>;
+	unauthorizedHeaders: Record<string, string>;
+	model: string;
+	prompt: string;
+	response: string;
+	contentType: string;
+	unauthorized: string;
+	invalid: string;
+};
+
+const fixtures: Fixture[] = [
+	{
+		mode: "openai",
+		url: (server) => `${server.baseUrl()}/chat/completions`,
+		headers: { authorization: "Bearer resolved-acme-key" },
+		unauthorizedHeaders: { authorization: "Bearer wrong-key" },
+		model: OPENAI_MODEL_ID,
+		prompt: OPENAI_PROBE_PROMPT,
+		response: OPENAI_PROBE_RESPONSE,
+		contentType: "text/event-stream",
+		unauthorized: "Invalid Acme credential",
+		invalid: "Invalid OpenAI-compatible request",
+	},
+	{
+		mode: "stream",
+		url: (server) => `${server.origin()}/generate`,
+		headers: { "x-acme-key": "resolved-stream-key" },
+		unauthorizedHeaders: { "x-acme-key": "wrong-key" },
+		model: STREAM_MODEL_ID,
+		prompt: STREAM_PROBE_PROMPT,
+		response: STREAM_PROBE_RESPONSE,
+		contentType: "application/x-ndjson",
+		unauthorized: "Invalid Acme Stream credential",
+		invalid: "Invalid Acme Stream request",
+	},
+];
+
+function body(model: string, prompt: string, stream = true) {
+	return { model, messages: [{ role: "user", content: prompt }], stream };
+}
+
+async function post(url: string, headers: Record<string, string>, payload: unknown) {
+	return fetch(url, {
+		method: "POST",
+		headers: { "content-type": "application/json", ...headers },
+		body: JSON.stringify(payload),
+	});
+}
+
+function streamedText(mode: Fixture["mode"], raw: string): string {
+	if (mode === "stream") {
+		return raw
+			.split("\n")
+			.filter(Boolean)
+			.map((line) => JSON.parse(line) as { type?: string; text?: string })
+			.filter((event) => event.type === "text_delta")
+			.map((event) => event.text ?? "")
+			.join("");
+	}
+	return raw
+		.split("\n")
+		.filter((line) => line.startsWith("data: ") && line !== "data: [DONE]")
+		.map((line) => JSON.parse(line.slice("data: ".length)) as { choices?: Array<{ delta?: { content?: string } }> })
+		.map((chunk) => chunk.choices?.[0]?.delta?.content ?? "")
+		.join("");
+}
+
+describe.each(fixtures)("$mode fixture", (fixture) => {
+	const server = createAcmeServer(fixture.mode);
+
+	beforeAll(() => server.start());
+	beforeEach(() => server.reset());
+	afterAll(() => server.stop());
+
+	it("accepts the probe request and records it", async () => {
+		const response = await post(fixture.url(server), fixture.headers, body(fixture.model, fixture.prompt));
+		expect(response.status).toBe(200);
+		expect(response.headers.get("content-type")).toContain(fixture.contentType);
+		expect(streamedText(fixture.mode, await response.text())).toBe(fixture.response);
+		expect(server.validRequestReceived()).toBe(true);
+	});
+
+	it("rejects a bad credential with 401 and does not record a probe", async () => {
+		const response = await post(
+			fixture.url(server),
+			fixture.unauthorizedHeaders,
+			body(fixture.model, fixture.prompt),
+		);
+		expect(response.status).toBe(401);
+		expect(await response.text()).toBe(fixture.unauthorized);
+		expect(server.validRequestReceived()).toBe(false);
+	});
+
+	it("rejects a malformed request with 422 and does not record a probe", async () => {
+		const response = await post(fixture.url(server), fixture.headers, body(fixture.model, fixture.prompt, false));
+		expect(response.status).toBe(422);
+		expect(await response.text()).toBe(fixture.invalid);
+		expect(server.validRequestReceived()).toBe(false);
+	});
+
+	it("does not treat a non-probe success as the probe", async () => {
+		const response = await post(fixture.url(server), fixture.headers, body(fixture.model, "hello"));
+		expect(response.status).toBe(200);
+		expect(server.validRequestReceived()).toBe(false);
+	});
+
+	it("clears the probe flag on reset", async () => {
+		await post(fixture.url(server), fixture.headers, body(fixture.model, fixture.prompt));
+		expect(server.validRequestReceived()).toBe(true);
+		server.reset();
+		expect(server.validRequestReceived()).toBe(false);
+	});
+});

third_party/pi/packages/evals/test/comparison.test.ts

@@ -0,0 +1,125 @@
+import { stripVTControlCharacters } from "node:util";
+import { describe, expect, it } from "vitest";
+import {
+	type EvalObservation,
+	type ExpectedEvalRun,
+	formatEvalComparisonReport,
+	summarizeEvalObservations,
+} from "../src/report.ts";
+
+function scored(
+	variant: EvalObservation["variant"],
+	runNumber: number,
+	score: number,
+	metrics: Partial<{
+		totalTokens: number;
+		toolCalls: number;
+		totalMs: number;
+		estimatedCostUsd: number;
+	}> = {},
+): EvalObservation {
+	return {
+		evalSet: "tool access",
+		caseId: "create",
+		variant,
+		model: "fixture/model",
+		runNumber,
+		outcome: "scored",
+		score,
+		totalTokens: 100,
+		toolCalls: 2,
+		totalMs: 1000,
+		estimatedCostUsd: 0.01,
+		...metrics,
+	};
+}
+
+function errored(variant: EvalObservation["variant"], runNumber: number, totalTokens?: number): EvalObservation {
+	return {
+		evalSet: "tool access",
+		caseId: "create",
+		variant,
+		model: "fixture/model",
+		runNumber,
+		outcome: "errored",
+		totalTokens,
+	};
+}
+
+function expectedFor(runNumbers: readonly number[]): ExpectedEvalRun[] {
+	return runNumbers.flatMap((runNumber) =>
+		(["without_docs", "with_docs"] as const).map((variant) => ({
+			evalSet: "tool access",
+			caseId: "create",
+			variant,
+			model: "fixture/model",
+			runNumber,
+		})),
+	);
+}
+
+describe("summarizeEvalObservations", () => {
+	it("computes paired lift and efficiency deltas", () => {
+		const observations = [
+			scored("without_docs", 1, 0, { totalTokens: 100, toolCalls: 3, totalMs: 1000 }),
+			scored("with_docs", 1, 1, { totalTokens: 120, toolCalls: 2, totalMs: 800 }),
+			scored("without_docs", 2, 1, { totalTokens: 200 }),
+			scored("with_docs", 2, 1, { totalTokens: 180 }),
+		];
+		const report = summarizeEvalObservations("digest", expectedFor([1, 2]), observations);
+		expect(report.comparisons).toEqual([
+			expect.objectContaining({
+				evalSet: "tool access",
+				totalPairs: 2,
+				eligiblePairs: 2,
+				blockedPairs: 0,
+				controlPassRate: 0.5,
+				treatmentPassRate: 1,
+				lift: 0.5,
+				totalTokens: { eligiblePairs: 2, controlMean: 150, treatmentMean: 150, meanDelta: 0 },
+				toolCalls: { eligiblePairs: 2, controlMean: 2.5, treatmentMean: 2, meanDelta: -0.5 },
+			}),
+		]);
+	});
+
+	it("fails closed for incomplete and errored pairs while retaining totals", () => {
+		const observations: EvalObservation[] = [
+			scored("without_docs", 1, 0),
+			errored("with_docs", 1, 120),
+			scored("without_docs", 2, 1, { totalTokens: 200 }),
+		];
+		const report = summarizeEvalObservations("digest", expectedFor([1, 2]), observations);
+		expect(report.comparisons[0]).toEqual(
+			expect.objectContaining({ totalPairs: 2, eligiblePairs: 0, blockedPairs: 2, lift: null }),
+		);
+		expect(report.blockedPairs).toEqual([
+			expect.objectContaining({ runNumber: 1, reasons: ["with_docs: errored"] }),
+			expect.objectContaining({ runNumber: 2, reasons: ["with_docs: expected 1 observation, found 0"] }),
+		]);
+		expect(report.operationalTotals[0].totalTokens).toEqual({ availableRuns: 2, total: 300 });
+	});
+
+	it("blocks duplicate observations and keeps missing metrics distinct from zero", () => {
+		const withoutTokens = scored("without_docs", 1, 1);
+		delete withoutTokens.totalTokens;
+		const report = summarizeEvalObservations("digest", expectedFor([1]), [
+			withoutTokens,
+			withoutTokens,
+			scored("with_docs", 1, 1, { totalTokens: 0 }),
+		]);
+		expect(report.blockedPairs[0].reasons).toEqual(["without_docs: expected 1 observation, found 2"]);
+		expect(report.operationalTotals[0].totalTokens).toEqual({ availableRuns: 0, total: null });
+		expect(report.operationalTotals[1].totalTokens).toEqual({ availableRuns: 1, total: 0 });
+	});
+
+	it("formats blocked comparisons and operational totals", () => {
+		const report = summarizeEvalObservations("digest", expectedFor([1, 2]), [
+			scored("without_docs", 1, 1),
+			scored("with_docs", 1, 1),
+		]);
+		const formatted = stripVTControlCharacters(formatEvalComparisonReport(report));
+		expect(formatted).toContain("Documentation Eval Comparisons");
+		expect(formatted).toContain("Pass rate  withheld because pairs are blocked");
+		expect(formatted).toContain("without_docs: 1 runs");
+	});
+});

third_party/pi/packages/evals/test/configured-runtime.test.ts

@@ -0,0 +1,196 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
+import {
+	createAcmeServer,
+	OPENAI_MODEL_ID,
+	OPENAI_PROBE_PROMPT,
+	OPENAI_PROBE_RESPONSE,
+	OPENAI_PROVIDER_ID,
+} from "../evals/acme-server.ts";
+import { inspectAddedModel, inspectProvider, loadConfiguredModelRuntime } from "../evals/configured-runtime.ts";
+
+const PROVIDER_ID = OPENAI_PROVIDER_ID;
+const MODEL = {
+	id: OPENAI_MODEL_ID,
+	name: "Acme Chat",
+	provider: PROVIDER_ID,
+	reasoning: false,
+	input: ["text"],
+	cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+	contextWindow: 32768,
+	maxTokens: 4096,
+};
+const server = createAcmeServer("openai");
+const temporaryDirectories: string[] = [];
+
+afterEach(async () => {
+	await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true })));
+});
+beforeAll(() => server.start());
+beforeEach(() => server.reset());
+afterAll(() => server.stop());
+
+async function agentDirWith(modelsJson: unknown) {
+	const directory = await mkdtemp(join(tmpdir(), "pi-eval-provider-probe-"));
+	temporaryDirectories.push(directory);
+	await writeFile(join(directory, "models.json"), `${JSON.stringify(modelsJson)}\n`);
+	return directory;
+}
+
+async function runtimeWith(modelsJson: unknown) {
+	return loadConfiguredModelRuntime(await agentDirWith(modelsJson));
+}
+
+function acmeModelsJson() {
+	return {
+		providers: {
+			[PROVIDER_ID]: {
+				baseUrl: server.baseUrl(),
+				api: "openai-completions",
+				apiKey: "$ACME_API_KEY",
+				models: [
+					{
+						id: MODEL.id,
+						name: MODEL.name,
+						reasoning: MODEL.reasoning,
+						input: MODEL.input,
+						cost: MODEL.cost,
+						contextWindow: MODEL.contextWindow,
+						maxTokens: MODEL.maxTokens,
+					},
+				],
+			},
+		},
+	};
+}
+
+function probe(content: string, apiKey: string) {
+	return {
+		providerId: PROVIDER_ID,
+		modelId: OPENAI_MODEL_ID,
+		createContext: () => ({
+			messages: [{ role: "user" as const, content, timestamp: 0 }],
+		}),
+		options: { env: { ACME_API_KEY: apiKey }, maxTokens: 32 },
+		validRequestReceived: server.validRequestReceived,
+	};
+}
+
+describe("inspectProvider", () => {
+	it("probes a models.json provider through ModelRuntime", async () => {
+		const runtime = await runtimeWith(acmeModelsJson());
+		await expect(inspectProvider(runtime, probe(OPENAI_PROBE_PROMPT, "resolved-acme-key"))).resolves.toEqual({
+			result: {
+				validRequestReceived: true,
+				model: MODEL,
+				response: { text: OPENAI_PROBE_RESPONSE, stopReason: "stop", inputTokens: 3, outputTokens: 2 },
+			},
+		});
+	});
+
+	it("keeps a non-probe completion distinct from a valid probe", async () => {
+		const runtime = await runtimeWith(acmeModelsJson());
+		await expect(inspectProvider(runtime, probe("hello", "resolved-acme-key"))).resolves.toEqual({
+			result: {
+				validRequestReceived: false,
+				model: MODEL,
+				response: { text: OPENAI_PROBE_RESPONSE, stopReason: "stop", inputTokens: 3, outputTokens: 2 },
+			},
+		});
+	});
+
+	it("returns a structured error when the configured model is missing", async () => {
+		const runtime = await runtimeWith({ providers: {} });
+		await expect(inspectProvider(runtime, probe(OPENAI_PROBE_PROMPT, "resolved-acme-key"))).resolves.toEqual({
+			result: { error: `Model ${PROVIDER_ID}/${OPENAI_MODEL_ID} is unavailable after reload.` },
+		});
+		expect(server.validRequestReceived()).toBe(false);
+	});
+
+	it("returns a structured error when models.json cannot be parsed", async () => {
+		const directory = await mkdtemp(join(tmpdir(), "pi-eval-provider-probe-"));
+		temporaryDirectories.push(directory);
+		await writeFile(join(directory, "models.json"), "{");
+		const output = await inspectProvider(
+			await loadConfiguredModelRuntime(directory),
+			probe(OPENAI_PROBE_PROMPT, "resolved-acme-key"),
+		);
+		expect(output).toEqual({
+			result: { error: expect.stringContaining("Failed to parse models.json") },
+		});
+		expect(server.validRequestReceived()).toBe(false);
+	});
+
+	it("does not treat an unauthorized completion as a valid probe", async () => {
+		const runtime = await runtimeWith(acmeModelsJson());
+		await expect(inspectProvider(runtime, probe(OPENAI_PROBE_PROMPT, "wrong-key"))).resolves.toEqual({
+			result: {
+				validRequestReceived: false,
+				model: MODEL,
+				response: { text: "", stopReason: "error", inputTokens: 0, outputTokens: 0 },
+			},
+		});
+	});
+});
+
+describe("inspectAddedModel", () => {
+	const PROVIDER = "openai";
+	const MODEL_ID = "fixture-chat";
+	const ADDED_MODEL = {
+		id: MODEL_ID,
+		name: "Fixture Chat",
+		provider: PROVIDER,
+		reasoning: true,
+		input: ["text"],
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+		contextWindow: 32768,
+		maxTokens: 4096,
+	};
+
+	it("loads an added model without dropping built-in models", async () => {
+		const directory = await agentDirWith({
+			providers: {
+				[PROVIDER]: {
+					models: [
+						{
+							id: ADDED_MODEL.id,
+							name: ADDED_MODEL.name,
+							reasoning: ADDED_MODEL.reasoning,
+							input: ADDED_MODEL.input,
+							cost: ADDED_MODEL.cost,
+							contextWindow: ADDED_MODEL.contextWindow,
+							maxTokens: ADDED_MODEL.maxTokens,
+						},
+					],
+				},
+			},
+		});
+		await expect(inspectAddedModel(await loadConfiguredModelRuntime(directory), PROVIDER, MODEL_ID)).resolves.toEqual(
+			{
+				result: { model: ADDED_MODEL, existingModelsPreserved: true },
+			},
+		);
+	});
+
+	it("returns a structured error when the added model is missing", async () => {
+		const directory = await agentDirWith({ providers: {} });
+		await expect(inspectAddedModel(await loadConfiguredModelRuntime(directory), PROVIDER, MODEL_ID)).resolves.toEqual(
+			{
+				result: { error: `Model ${PROVIDER}/${MODEL_ID} is unavailable after reload.` },
+			},
+		);
+	});
+
+	it("returns a structured error when models.json cannot be parsed", async () => {
+		const directory = await mkdtemp(join(tmpdir(), "pi-eval-provider-probe-"));
+		temporaryDirectories.push(directory);
+		await writeFile(join(directory, "models.json"), "{");
+		await expect(inspectAddedModel(await loadConfiguredModelRuntime(directory), PROVIDER, MODEL_ID)).resolves.toEqual(
+			{
+				result: { error: expect.stringContaining("Failed to parse models.json") },
+			},
+		);
+	});
+});

third_party/pi/packages/evals/test/harness.test.ts

@@ -0,0 +1,116 @@
+import { homedir } from "node:os";
+import { getDocsPath, getExamplesPath, getReadmePath } from "@earendil-works/pi-coding-agent";
+import { describe, expect, it, vi } from "vitest";
+import { buildSystemPrompt } from "../../coding-agent/src/core/system-prompt.ts";
+import {
+	applyIsolatedEnvironment,
+	createPiDocumentationEvalHarness,
+	DOCUMENTATION_EVAL_TOOLS,
+	excludePiDocumentation,
+	resolveDocumentationVariant,
+	resolveModelSelection,
+	verifySystemPrompt,
+} from "../src/harness.ts";
+
+describe("resolveModelSelection", () => {
+	it("prefers an explicit harness model", () => {
+		expect(
+			resolveModelSelection(
+				{ provider: "anthropic", id: "claude-opus-4-6" },
+				{ PI_PROVIDER: "openai-codex", PI_MODEL: "gpt-5.6-sol" },
+			),
+		).toEqual({ provider: "anthropic", id: "claude-opus-4-6" });
+	});
+
+	it("uses trimmed environment defaults", () => {
+		expect(resolveModelSelection(undefined, { PI_PROVIDER: " openai-codex ", PI_MODEL: " gpt-5.6-sol " })).toEqual({
+			provider: "openai-codex",
+			id: "gpt-5.6-sol",
+		});
+	});
+
+	it.each([{}, { PI_PROVIDER: "openai-codex" }, { PI_MODEL: "gpt-5.6-sol" }])(
+		"rejects incomplete model selection",
+		(environment) => {
+			expect(() => resolveModelSelection(undefined, environment)).toThrow("Select a harness model explicitly");
+		},
+	);
+});
+
+describe("isolateProcessEnvironment", () => {
+	it("removes runner metadata and restores the process environment", () => {
+		vi.stubEnv("PI_EVAL_VARIANT", "with_docs");
+		vi.stubEnv("PI_EVAL_ARTIFACT_DIR", "/tmp/artifacts");
+		const oldHome = process.env.HOME;
+		try {
+			const restore = applyIsolatedEnvironment("/tmp/eval-home", "/tmp/eval-agent");
+			try {
+				expect(homedir()).toBe("/tmp/eval-home");
+				expect(process.env.PI_CODING_AGENT_DIR).toBe("/tmp/eval-agent");
+				expect(process.env.PI_EVAL_VARIANT).toBeUndefined();
+				expect(process.env.PI_EVAL_ARTIFACT_DIR).toBeUndefined();
+			} finally {
+				restore();
+			}
+			expect(process.env.HOME).toBe(oldHome);
+			expect(process.env.PI_EVAL_VARIANT).toBe("with_docs");
+		} finally {
+			vi.unstubAllEnvs();
+		}
+	});
+});
+
+describe("documentation variant", () => {
+	it.each(["without_docs", "with_docs"] as const)("accepts %s", (variant) => {
+		expect(resolveDocumentationVariant(variant)).toBe(variant);
+	});
+
+	it.each([undefined, "", "other"])("rejects invalid variant %s", (variant) => {
+		expect(() => resolveDocumentationVariant(variant)).toThrow("PI_EVAL_VARIANT");
+	});
+
+	it("strips only the documentation routing section from the default Pi prompt", () => {
+		const prompt = buildSystemPrompt({
+			cwd: "/workspace",
+			selectedTools: [...DOCUMENTATION_EVAL_TOOLS],
+		});
+		expect(prompt).toContain("\n<docs>\nPi documentation (read only");
+		expect(prompt).toContain("\n<rules>\n");
+		expect(prompt).toContain("\n<cwd>\n/workspace\n</cwd>");
+		expect(prompt).toContain("docs/models.md");
+
+		const stripped = excludePiDocumentation(prompt);
+		expect(stripped).toContain("\n<rules>\n");
+		expect(stripped).toContain("\n<cwd>\n/workspace\n</cwd>");
+		expect(stripped).not.toContain("<docs>");
+		expect(stripped).not.toContain("Pi documentation");
+		expect(stripped).not.toContain("docs/models.md");
+		expect(stripped).not.toContain(getReadmePath());
+		expect(stripped).not.toContain(getDocsPath());
+		expect(stripped).not.toContain(getExamplesPath());
+	});
+
+	it("verifies the prompt that was sent", () => {
+		const prompt = buildSystemPrompt({
+			cwd: "/workspace",
+			selectedTools: [...DOCUMENTATION_EVAL_TOOLS],
+		});
+		const stripped = excludePiDocumentation(prompt);
+
+		expect(verifySystemPrompt(stripped, { name: "without_docs", expectedPiDocumentation: false })).toBe(stripped);
+		expect(() => verifySystemPrompt(prompt, { name: "without_docs", expectedPiDocumentation: false })).toThrow(
+			"does not match",
+		);
+	});
+
+	it("fails closed when prompt markers are missing", () => {
+		expect(() => excludePiDocumentation("Instructions")).toThrow("no Pi documentation section");
+		expect(() => excludePiDocumentation("\n<docs>\nPi documentation\n</docs>")).toThrow(
+			"no working-directory section",
+		);
+	});
+
+	it("rejects documentation harnesses outside the container sandbox", () => {
+		expect(() => createPiDocumentationEvalHarness()).toThrow("isolated container sandbox");
+	});
+});

third_party/pi/packages/evals/test/pi-harness.test.ts

@@ -1,34 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { resolveModelSelection } from "../src/pi-harness.ts";
-
-describe("resolveModelSelection", () => {
-	it("prefers an explicit harness model over environment defaults", () => {
-		expect(
-			resolveModelSelection(
-				{ provider: "anthropic", id: "claude-opus-4-6" },
-				{ PI_PROVIDER: "openai-codex", PI_MODEL: "gpt-5.6-sol" },
-			),
-		).toEqual({ provider: "anthropic", id: "claude-opus-4-6" });
-	});
-
-	it("uses trimmed environment defaults when the harness has no explicit model", () => {
-		expect(resolveModelSelection(undefined, { PI_PROVIDER: " openai-codex ", PI_MODEL: " gpt-5.6-sol " })).toEqual({
-			provider: "openai-codex",
-			id: "gpt-5.6-sol",
-		});
-	});
-
-	it.each([
-		[undefined, {}],
-		[undefined, { PI_PROVIDER: "openai-codex" }],
-		[undefined, { PI_MODEL: "gpt-5.6-sol" }],
-		[
-			{ provider: "", id: "gpt-5.6-sol" },
-			{ PI_PROVIDER: "openai-codex", PI_MODEL: "gpt-5.6-sol" },
-		],
-	] as const)("rejects an incomplete model selection", (explicitModel, environment) => {
-		expect(() => resolveModelSelection(explicitModel, environment)).toThrow(
-			"Select a harness model explicitly or set both PI_PROVIDER and PI_MODEL as defaults.",
-		);
-	});
-});

third_party/pi/packages/evals/test/plan.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import { createTaskPlan, parseDiscoveredCases } from "../src/plan.ts";
+
+const discovered = [{ name: "Add model > adds the model", file: "evals/models.docs.eval.ts" }];
+
+describe("parseDiscoveredCases", () => {
+	it("derives stable case identity from ordinary Vitest names", () => {
+		expect(parseDiscoveredCases(discovered)).toEqual([
+			{
+				fullName: "Add model > adds the model",
+				evalSet: "Add model",
+				caseId: "adds the model",
+				file: "evals/models.docs.eval.ts",
+			},
+		]);
+	});
+
+	it("rejects ambiguous and duplicate identities", () => {
+		expect(() => parseDiscoveredCases([{ name: "adds the model", file: "model.ts" }])).toThrow("<eval set> > <case>");
+		expect(() => parseDiscoveredCases([...discovered, ...discovered])).toThrow("Duplicate eval case identity");
+	});
+});
+
+describe("createTaskPlan", () => {
+	it("creates one isolated task per case, variant, model, and repetition", () => {
+		const cases = parseDiscoveredCases(discovered);
+		const tasks = createTaskPlan(cases, "fixture/model", 2);
+		expect(tasks).toHaveLength(4);
+		expect(tasks.map(({ variant, runNumber }) => [variant, runNumber])).toEqual([
+			["without_docs", 1],
+			["with_docs", 1],
+			["with_docs", 2],
+			["without_docs", 2],
+		]);
+	});
+
+	it("rejects invalid model identities and repetitions", () => {
+		const cases = parseDiscoveredCases(discovered);
+		expect(() => createTaskPlan(cases, "model", 1)).toThrow("provider and model");
+		expect(() => createTaskPlan(cases, "fixture/model", 0)).toThrow("positive integer");
+	});
+});

third_party/pi/packages/evals/test/report.test.ts

@@ -0,0 +1,208 @@
+import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import type { EvalTask } from "../src/plan.ts";
+import { classifyCaseStatus, PI_SESSION_SNAPSHOT_ARTIFACT, readTaskObservation } from "../src/report.ts";
+
+const temporaryDirectories: string[] = [];
+afterEach(async () => {
+	await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { recursive: true, force: true })));
+});
+
+const task: EvalTask = {
+	file: "evals/example.docs.eval.ts",
+	fullName: "Example workflow > handles the case",
+	evalSet: "Example workflow",
+	caseId: "handles the case",
+	variant: "without_docs",
+	model: "fixture/model",
+	runNumber: 1,
+};
+const SESSION = '{"type":"session"}\n';
+
+type ReportAssertion = {
+	status?: "passed" | "skipped" | "pending" | "failed";
+	meta?: Record<string, unknown>;
+};
+
+async function writeTaskReport(assertion: ReportAssertion = {}) {
+	const directory = await mkdtemp(join(tmpdir(), "pi-eval-report-test-"));
+	temporaryDirectories.push(directory);
+	const reportPath = join(directory, "vitest.json");
+	const status = assertion.status ?? "passed";
+	await writeFile(
+		reportPath,
+		JSON.stringify({
+			numFailedTests: 0,
+			numPassedTests: status === "passed" ? 1 : 0,
+			numPendingTests: status === "pending" || status === "skipped" ? 1 : 0,
+			numTodoTests: 0,
+			numTotalTests: 1,
+			startTime: 0,
+			success: true,
+			testResults: [
+				{
+					message: "",
+					name: "/repo/packages/evals/evals/example.docs.eval.ts",
+					status: "passed",
+					assertionResults: [
+						{
+							ancestorTitles: [task.evalSet],
+							fullName: `${task.evalSet} ${task.caseId}`,
+							status,
+							title: task.caseId,
+							failureMessages: [],
+							meta: assertion.meta ?? {},
+						},
+					],
+				},
+			],
+		}),
+	);
+	return { directory, reportPath };
+}
+
+function scoredMeta(overrides?: {
+	avgScore?: number | null;
+	model?: string;
+	errors?: Array<Record<string, string>>;
+	artifacts?: Record<string, string>;
+}): Record<string, unknown> {
+	return {
+		eval: {
+			avgScore: overrides && "avgScore" in overrides ? overrides.avgScore : 0.5,
+			scores: [{ name: "StructuredOutputJudge", score: 0.5 }],
+			thresholdFailed: false,
+		},
+		harness: {
+			name: "without_docs",
+			run: {
+				output: { ok: true },
+				session: { events: [{ type: "message", role: "user", content: "prompt" }] },
+				usage: {
+					provider: "fixture",
+					model: overrides?.model ?? "model",
+					inputTokens: 10,
+					outputTokens: 5,
+					totalTokens: 15,
+					toolCalls: 1,
+					metadata: { cacheReadTokens: 2, cacheWriteTokens: 3, estimatedCostUsd: 0.01 },
+				},
+				timings: { totalMs: 1234 },
+				artifacts: overrides?.artifacts ?? { runId: "run-1", [PI_SESSION_SNAPSHOT_ARTIFACT]: SESSION },
+				errors: overrides?.errors ?? [],
+			},
+		},
+	};
+}
+
+async function readObservation(assertion?: ReportAssertion) {
+	const { directory, reportPath } = await writeTaskReport(assertion);
+	return { directory, observation: await readTaskObservation(task, reportPath, directory) };
+}
+
+describe("classifyCaseStatus", () => {
+	it.each(["skipped", "todo", "disabled"] as const)("maps %s to skipped", (status) => {
+		expect(classifyCaseStatus(status)).toBe("skipped");
+	});
+
+	it("maps failed infrastructure to errored", () => {
+		expect(classifyCaseStatus("failed")).toBe("errored");
+	});
+});
+
+describe("readTaskObservation", () => {
+	it("preserves a skipped outcome when no harness run exists", async () => {
+		const { observation } = await readObservation({ status: "skipped" });
+		expect(observation).toMatchObject({ outcome: "skipped" });
+	});
+
+	it("preserves a pending outcome when no harness run exists", async () => {
+		const { observation } = await readObservation({ status: "pending" });
+		expect(observation).toMatchObject({ outcome: "pending" });
+	});
+
+	it("preserves metrics from a failed eval with a partial harness run", async () => {
+		const { directory, observation } = await readObservation({
+			status: "failed",
+			meta: scoredMeta({ errors: [{ message: "Prompt verification failed" }] }),
+		});
+		expect(observation).toEqual({
+			evalSet: task.evalSet,
+			caseId: task.caseId,
+			variant: task.variant,
+			model: task.model,
+			runNumber: task.runNumber,
+			outcome: "errored",
+			inputTokens: 10,
+			outputTokens: 5,
+			cacheReadTokens: 2,
+			cacheWriteTokens: 3,
+			totalTokens: 15,
+			toolCalls: 1,
+			totalMs: 1234,
+			estimatedCostUsd: 0.01,
+		});
+		const hashes = await readdir(join(directory, task.variant, "sessions"));
+		await expect(
+			readFile(join(directory, task.variant, "sessions", hashes[0], "session.jsonl"), "utf8"),
+		).resolves.toBe(SESSION);
+	});
+
+	it("records an errored outcome when a passed eval has no harness run", async () => {
+		const { observation } = await readObservation({ status: "passed" });
+		expect(observation).toMatchObject({ outcome: "errored" });
+	});
+
+	it("reads a scored harness run and persists the session artifact", async () => {
+		const { directory, observation } = await readObservation({ meta: scoredMeta() });
+		expect(observation).toEqual({
+			evalSet: task.evalSet,
+			caseId: task.caseId,
+			variant: task.variant,
+			model: task.model,
+			runNumber: task.runNumber,
+			outcome: "scored",
+			score: 0.5,
+			inputTokens: 10,
+			outputTokens: 5,
+			cacheReadTokens: 2,
+			cacheWriteTokens: 3,
+			totalTokens: 15,
+			toolCalls: 1,
+			totalMs: 1234,
+			estimatedCostUsd: 0.01,
+		});
+		const hashes = await readdir(join(directory, task.variant, "sessions"));
+		expect(hashes).toHaveLength(1);
+		await expect(
+			readFile(join(directory, task.variant, "sessions", hashes[0], "session.jsonl"), "utf8"),
+		).resolves.toBe(SESSION);
+	});
+
+	it("treats a zero score as scored data", async () => {
+		const { observation } = await readObservation({ meta: scoredMeta({ avgScore: 0 }) });
+		expect(observation).toMatchObject({ outcome: "scored", score: 0 });
+	});
+
+	it("records an unscored outcome when a completed eval has no score", async () => {
+		const { observation } = await readObservation({ meta: scoredMeta({ avgScore: null }) });
+		expect(observation).toMatchObject({ outcome: "unscored" });
+	});
+
+	it("records an errored outcome when the reported model does not match the task", async () => {
+		const { observation } = await readObservation({ meta: scoredMeta({ model: "other" }) });
+		expect(observation).toMatchObject({ outcome: "errored" });
+	});
+
+	it("records an errored outcome when a completed harness run contains errors", async () => {
+		const { observation } = await readObservation({ meta: scoredMeta({ errors: [{ message: "boom" }] }) });
+		expect(observation).toMatchObject({ outcome: "errored" });
+	});
+
+	it("still scores a completed eval when the session artifact is missing", async () => {
+		const { observation } = await readObservation({ meta: scoredMeta({ artifacts: { runId: "run-1" } }) });
+		expect(observation).toMatchObject({ outcome: "scored", score: 0.5 });
+	});
+});

third_party/pi/packages/evals/test/vitest-evals/artifacts.test.ts

@@ -1,103 +0,0 @@
-import { mkdtemp, readFile, rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { expect, it } from "vitest";
-import {
-	persistEvalArtifactReferences,
-	recordEvalSessionArtifact,
-	recordEvalSourceArtifact,
-} from "../../src/vitest-evals/artifacts.ts";
-
-it("records session and source artifacts against the explicit test task", async ({ task }) => {
-	const runId = "run-1";
-	await recordEvalSessionArtifact(task, {
-		artifacts: { runId, piSessionJsonl: '{"type":"session"}\n' },
-	});
-	await recordEvalSourceArtifact(task, runId, {
-		name: "hello.ts",
-		contentType: "text/typescript",
-		body: "export default function () {}\n",
-		bodyEncoding: "utf-8",
-	});
-
-	expect(task.artifacts).toContainEqual(
-		expect.objectContaining({
-			type: "@earendil-works/pi-evals:session",
-			runId,
-			attachments: [
-				expect.objectContaining({
-					name: "session.jsonl",
-					body: '{"type":"session"}\n',
-					bodyEncoding: "utf-8",
-					contentType: "application/jsonl",
-				}),
-			],
-		}),
-	);
-	expect(task.artifacts).toContainEqual(
-		expect.objectContaining({
-			type: "@earendil-works/pi-evals:source",
-			runId,
-			attachments: [
-				expect.objectContaining({
-					name: "hello.ts",
-					body: "export default function () {}\n",
-					bodyEncoding: "utf-8",
-					contentType: "text/typescript",
-				}),
-			],
-		}),
-	);
-});
-
-it("persists and selects attachments belonging to the reported run", async () => {
-	const root = await mkdtemp(join(tmpdir(), "pi-eval-artifact-report-test-"));
-	try {
-		const references = await persistEvalArtifactReferences(
-			[
-				{
-					type: "@earendil-works/pi-evals:session",
-					runId: "run-1",
-					attachments: [
-						{
-							name: "session.jsonl",
-							body: '{"type":"session"}\n',
-							bodyEncoding: "utf-8",
-							contentType: "application/jsonl",
-						},
-					],
-				},
-				{
-					type: "@earendil-works/pi-evals:session",
-					runId: "run-2",
-					attachments: [],
-				},
-				{
-					type: "@earendil-works/pi-evals:source",
-					runId: "run-1",
-					attachments: [
-						{
-							name: "hello.ts",
-							body: "export default function () {}\n",
-							bodyEncoding: "utf-8",
-							contentType: "text/typescript",
-						},
-					],
-				},
-				{ type: "internal:annotation", annotation: { message: "other", type: "info" } },
-			],
-			"run-1",
-			root,
-		);
-		expect(references).toEqual([
-			{ name: "session.jsonl", path: expect.stringMatching(/^sessions\/[a-f0-9]{64}\/session\.jsonl$/) },
-			{ name: "hello.ts", path: expect.stringMatching(/^sources\/[a-f0-9]{64}\/hello\.ts$/) },
-		]);
-		for (const { name, path } of references) {
-			const expected = name === "session.jsonl" ? '{"type":"session"}\n' : "export default function () {}\n";
-			expect(await readFile(join(root, path), "utf8")).toBe(expected);
-		}
-	} finally {
-		await rm(root, { recursive: true, force: true });
-	}
-});

third_party/pi/packages/evals/test/vitest-evals/harness-table.test.ts

@@ -1,94 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { createHarness, type HarnessContext } from "vitest-evals/harness";
-import {
-	deriveEvalGroupKey,
-	EVAL_HARNESS_ITERATION_ARTIFACT,
-	evalHarnessTable,
-	parseEvalHarnessIterationArtifact,
-} from "../../src/vitest-evals/harness-table.ts";
-
-describe("deriveEvalGroupKey", () => {
-	it("combines a trimmed string input ID with repetition", () => {
-		expect(deriveEvalGroupKey({ id: " input-1 ", prompt: "hello" }, 2)).toBe(JSON.stringify(["input-1", 2]));
-	});
-
-	it("hashes canonical JSON independently of object key order", () => {
-		expect(deriveEvalGroupKey({ first: 1, second: [true, "value"] }, 1)).toBe(
-			deriveEvalGroupKey({ second: [true, "value"], first: 1 }, 1),
-		);
-		expect(deriveEvalGroupKey({ first: 1 }, 1)).not.toBe(deriveEvalGroupKey({ first: 2 }, 1));
-		expect(deriveEvalGroupKey({ first: 1 }, 1)).not.toBe(deriveEvalGroupKey({ first: 1 }, 2));
-		expect(deriveEvalGroupKey(["first", "second"], 1)).not.toBe(deriveEvalGroupKey(["second", "first"], 1));
-	});
-
-	it("rejects non-JSON and circular input", () => {
-		const circular: { self?: unknown } = {};
-		circular.self = circular;
-		expect(() => deriveEvalGroupKey(new Date(0), 1)).toThrow("only plain objects and arrays");
-		expect(() => deriveEvalGroupKey(Array(1), 1)).toThrow("must not be sparse");
-		expect(() => deriveEvalGroupKey(circular, 1)).toThrow("must not contain circular references");
-	});
-});
-
-function createFakeHarness(name: string) {
-	return createHarness<{ id: string }, { harness: string; inputId: string }>({
-		name,
-		run: ({ input }) => ({
-			output: { harness: name, inputId: input.id },
-			events: [
-				{ type: "message", role: "user", content: input.id },
-				{ type: "message", role: "assistant", content: name },
-			],
-		}),
-	});
-}
-
-const harnessTable = evalHarnessTable("local multi-harness eval", {
-	baseline: createFakeHarness("withoutSkill"),
-	candidates: [createFakeHarness("withSkill")],
-	repetitions: 2,
-});
-
-describe("evalHarnessTable", () => {
-	it("plans repetitions in declaration order", () => {
-		expect(harnessTable.map(({ name, repetition }) => ({ name, repetition }))).toEqual([
-			{ name: "withoutSkill", repetition: 1 },
-			{ name: "withSkill", repetition: 1 },
-			{ name: "withoutSkill", repetition: 2 },
-			{ name: "withSkill", repetition: 2 },
-		]);
-	});
-
-	it("accepts a singular candidate", () => {
-		const rows = evalHarnessTable("singular candidate", {
-			baseline: createFakeHarness("baseline"),
-			candidate: createFakeHarness("candidate"),
-		});
-
-		expect(rows.map(({ name }) => name)).toEqual(["baseline", "candidate"]);
-	});
-
-	it("attaches iteration metadata to every wrapped harness run", async () => {
-		for (const row of harnessTable) {
-			const artifacts: HarnessContext["artifacts"] = {};
-			const context: HarnessContext = {
-				artifacts,
-				setArtifact(name, value) {
-					artifacts[name] = value;
-				},
-			};
-			const result = await row.harness.run({ id: "first" }, context);
-
-			expect(result.output).toEqual({ harness: row.name, inputId: "first" });
-			expect(parseEvalHarnessIterationArtifact(result.artifacts?.[EVAL_HARNESS_ITERATION_ARTIFACT])).toEqual({
-				schemaVersion: 1,
-				evalSet: "local multi-harness eval",
-				groupKey: deriveEvalGroupKey({ id: "first" }, row.repetition),
-				harness: row.name,
-				baseline: "withoutSkill",
-				candidates: ["withSkill"],
-				repetition: row.repetition,
-			});
-		}
-	});
-});

third_party/pi/packages/evals/test/vitest-evals/summary.test.ts

@@ -1,232 +0,0 @@
-import { stripVTControlCharacters } from "node:util";
-import { describe, expect, it } from "vitest";
-import {
-	formatHarnessComparisonReport,
-	type HarnessObservation,
-	summarizeHarnessComparisons,
-} from "../../src/vitest-evals/summary.ts";
-
-type ObservationResult = "passed" | "failed" | Exclude<HarnessObservation["outcome"], "scored">;
-
-function observation(
-	harness: string,
-	testName: string,
-	result: ObservationResult,
-	metrics: Pick<HarnessObservation, "totalTokens" | "totalMs" | "estimatedCostUsd"> = {},
-	baseline = "without-tools",
-	candidates: string[] = ["with-tools"],
-): HarnessObservation {
-	const base = {
-		evalSet: "tool access",
-		groupKey: JSON.stringify([testName, 1]),
-		testName,
-		file: "src/tool-access.eval.ts",
-		harness,
-		baseline,
-		candidates,
-		repetition: 1,
-		...metrics,
-	};
-	if (result === "passed" || result === "failed") {
-		return { ...base, outcome: "scored", score: result === "passed" ? 1 : 0 };
-	}
-	return { ...base, outcome: result };
-}
-
-describe("summarizeHarnessComparisons", () => {
-	it("computes paired correctness lift separately from efficiency deltas", () => {
-		const report = summarizeHarnessComparisons([
-			observation("without-tools", "create", "failed", {
-				totalTokens: 100,
-				totalMs: 1000,
-				estimatedCostUsd: 0.01,
-			}),
-			observation("with-tools", "create", "passed", {
-				totalTokens: 120,
-				totalMs: 800,
-				estimatedCostUsd: 0.02,
-			}),
-			observation("without-tools", "inspect", "passed", { totalTokens: 200 }),
-			observation("with-tools", "inspect", "passed", { totalTokens: 180 }),
-		]);
-
-		expect(report.evalSets).toHaveLength(1);
-		expect(report.evalSets[0]?.comparisons).toEqual([
-			expect.objectContaining({
-				baseline: "without-tools",
-				candidate: "with-tools",
-				correctness: {
-					totalPairs: 2,
-					eligiblePairs: 2,
-					baselinePassRate: 0.5,
-					candidatePassRate: 1,
-					lift: 0.5,
-					baselineWins: 0,
-					candidateWins: 1,
-					ties: 1,
-				},
-				totalTokens: {
-					totalPairs: 2,
-					eligiblePairs: 2,
-					baselineMean: 150,
-					candidateMean: 150,
-					meanDelta: 0,
-				},
-				totalMs: {
-					totalPairs: 2,
-					eligiblePairs: 1,
-					baselineMean: 1000,
-					candidateMean: 800,
-					meanDelta: -200,
-				},
-				estimatedCostUsd: {
-					totalPairs: 2,
-					eligiblePairs: 1,
-					baselineMean: 0.01,
-					candidateMean: 0.02,
-					meanDelta: 0.01,
-				},
-			}),
-		]);
-		expect(report.diagnostics).toEqual([]);
-	});
-
-	it("reports missing observations without coercing them to failures or zero telemetry", () => {
-		const report = summarizeHarnessComparisons([
-			observation("without-tools", "create", "failed"),
-			observation("with-tools", "create", "passed"),
-			observation("without-tools", "inspect", "passed"),
-		]);
-		const comparison = report.evalSets[0]?.comparisons[0];
-
-		expect(comparison?.correctness).toEqual({
-			totalPairs: 2,
-			eligiblePairs: 1,
-			baselinePassRate: 0,
-			candidatePassRate: 1,
-			lift: 1,
-			baselineWins: 0,
-			candidateWins: 1,
-			ties: 0,
-		});
-		expect(comparison?.totalTokens).toEqual({
-			totalPairs: 2,
-			eligiblePairs: 0,
-			baselineMean: null,
-			candidateMean: null,
-			meanDelta: null,
-		});
-		expect(report.diagnostics).toContainEqual(
-			expect.objectContaining({
-				testName: "inspect",
-				harness: "with-tools",
-				reason: "missing-observation",
-			}),
-		);
-	});
-
-	it("keeps identical inputs in different test files separate", () => {
-		const report = summarizeHarnessComparisons([
-			observation("without-tools", "shared", "failed"),
-			observation("with-tools", "shared", "passed"),
-			{ ...observation("without-tools", "shared", "passed"), file: "src/other.eval.ts" },
-			{ ...observation("with-tools", "shared", "passed"), file: "src/other.eval.ts" },
-		]);
-
-		expect(report.evalSets[0]?.comparisons[0]?.correctness).toEqual(
-			expect.objectContaining({ totalPairs: 2, eligiblePairs: 2 }),
-		);
-		expect(report.diagnostics).toEqual([]);
-	});
-
-	it("does not score harness errors as correctness failures", () => {
-		const report = summarizeHarnessComparisons([
-			observation("without-tools", "create", "errored", { totalTokens: 100 }),
-			observation("with-tools", "create", "passed", { totalTokens: 100 }),
-		]);
-
-		expect(report.evalSets[0]?.comparisons[0]?.correctness).toEqual(
-			expect.objectContaining({ totalPairs: 1, eligiblePairs: 0 }),
-		);
-		expect(report.evalSets[0]?.comparisons[0]?.totalTokens.eligiblePairs).toBe(0);
-		expect(report.diagnostics).toContainEqual(
-			expect.objectContaining({ harness: "without-tools", reason: "harness-error" }),
-		);
-	});
-
-	it("does not derive correctness from completed Vitest tests without judge scores", () => {
-		const withoutScore = observation("without-tools", "create", "unscored");
-		const withScore = observation("with-tools", "create", "unscored");
-
-		const report = summarizeHarnessComparisons([withoutScore, withScore]);
-
-		expect(report.evalSets[0]?.comparisons[0]?.correctness.eligiblePairs).toBe(0);
-		expect(report.diagnostics).toEqual([
-			expect.objectContaining({ harness: "with-tools", reason: "missing-score" }),
-			expect.objectContaining({ harness: "without-tools", reason: "missing-score" }),
-		]);
-	});
-
-	it("compares each candidate with the declared baseline", () => {
-		const candidates = ["second", "third"];
-		const report = summarizeHarnessComparisons([
-			observation("first", "input", "passed", {}, "first", candidates),
-			observation("second", "input", "passed", {}, "first", candidates),
-			observation("third", "input", "passed", {}, "first", candidates),
-		]);
-
-		expect(report.evalSets[0]?.comparisons.map(({ baseline, candidate }) => [baseline, candidate])).toEqual([
-			["first", "second"],
-			["first", "third"],
-		]);
-	});
-
-	it("retains a declared harness with no completed observations", () => {
-		const report = summarizeHarnessComparisons([observation("without-tools", "create", "failed")]);
-
-		expect(report.evalSets[0]?.comparisons).toHaveLength(1);
-		expect(report.evalSets[0]?.comparisons[0]?.correctness.eligiblePairs).toBe(0);
-		expect(report.diagnostics).toContainEqual(
-			expect.objectContaining({
-				testName: "create",
-				harness: "with-tools",
-				reason: "missing-observation",
-			}),
-		);
-	});
-
-	it("reports duplicate and unscorable observations once across multiple harness pairs", () => {
-		const candidates = ["second", "third"];
-		const report = summarizeHarnessComparisons([
-			observation("first", "duplicate", "passed", {}, "first", candidates),
-			observation("first", "duplicate", "failed", {}, "first", candidates),
-			observation("second", "duplicate", "passed", {}, "first", candidates),
-			observation("third", "duplicate", "passed", {}, "first", candidates),
-			observation("first", "skipped", "skipped", {}, "first", candidates),
-			observation("second", "skipped", "passed", {}, "first", candidates),
-			observation("third", "skipped", "passed", {}, "first", candidates),
-		]);
-
-		expect(report.diagnostics.filter(({ reason }) => reason === "duplicate-observation")).toEqual([
-			expect.objectContaining({ testName: "duplicate", harness: "first" }),
-		]);
-		expect(report.diagnostics.filter(({ reason }) => reason === "unscorable-outcome")).toEqual([
-			expect.objectContaining({ testName: "skipped", harness: "first" }),
-		]);
-	});
-
-	it("formats lift and telemetry availability for the terminal report", () => {
-		const report = summarizeHarnessComparisons([
-			observation("without-tools", "create", "failed", { totalMs: 34853.7 }),
-			observation("with-tools", "create", "passed", { totalMs: 30694.2 }),
-		]);
-
-		const formatted = stripVTControlCharacters(formatHarnessComparisonReport(report));
-		expect(formatted).toContain("Eval Comparisons");
-		expect(formatted).toContain(" Baseline  without-tools");
-		expect(formatted).toContain("Candidate  with-tools (1/1 pairs)");
-		expect(formatted).toContain("Pass rate  +100.0 pp (candidate 100.0%, baseline 0.0%)");
-		expect(formatted).toContain("   Tokens  unavailable");
-		expect(formatted).toContain("  Latency  -4159.5ms (candidate 30694.2ms, baseline 34853.7ms)");
-	});
-});