chore(vendor): import pi 1.3.0
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree. git-subtree-dir: third_party/pi git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
This commit is contained in:
192
third_party/pi/packages/evals/src/cli.ts
vendored
Normal file
192
third_party/pi/packages/evals/src/cli.ts
vendored
Normal file
@@ -0,0 +1,192 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { globSync } from "node:fs";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { dirname, relative, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { buildImages, createDockerContext, discoverCases, requireEvalAuthFile, runTask } from "./docker.ts";
|
||||
import { createTaskPlan, type DiscoveredEvalCase, DOCUMENTATION_VARIANTS, parseDiscoveredCases } from "./plan.ts";
|
||||
import {
|
||||
type EvalObservation,
|
||||
erroredObservation,
|
||||
formatEvalComparisonReport,
|
||||
readTaskObservation,
|
||||
summarizeEvalObservations,
|
||||
} from "./report.ts";
|
||||
|
||||
type EvalCliOptions = {
|
||||
provider?: string;
|
||||
model?: string;
|
||||
runsPerVariant: number;
|
||||
requestedFiles: string[];
|
||||
discoveryArgs: string[];
|
||||
};
|
||||
|
||||
const RUNNER_OPTIONS = new Set(["--provider", "--model", "--runs-per-variant"]);
|
||||
const FILTER_OPTIONS = new Set(["-t", "--testNamePattern"]);
|
||||
|
||||
function parseEvalCli(args: readonly string[], environment: NodeJS.ProcessEnv = process.env): EvalCliOptions {
|
||||
let provider: string | undefined;
|
||||
let model: string | undefined;
|
||||
let runsText: string | undefined;
|
||||
let cliSelectedModel = false;
|
||||
const requestedFiles: string[] = [];
|
||||
const discoveryArgs: string[] = [];
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const argument = args[index];
|
||||
if (argument.endsWith(".docs.eval.ts")) {
|
||||
requestedFiles.push(argument);
|
||||
continue;
|
||||
}
|
||||
if (FILTER_OPTIONS.has(argument)) {
|
||||
const value = args[index + 1];
|
||||
if (!value) throw new Error(`Missing value for ${argument}.`);
|
||||
discoveryArgs.push(argument, value);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (argument.startsWith("--testNamePattern=")) {
|
||||
if (argument.length === "--testNamePattern=".length) throw new Error("Missing value for --testNamePattern.");
|
||||
discoveryArgs.push(argument);
|
||||
continue;
|
||||
}
|
||||
|
||||
const equals = argument.indexOf("=");
|
||||
const name = equals === -1 ? argument : argument.slice(0, equals);
|
||||
if (RUNNER_OPTIONS.has(name)) {
|
||||
const value = equals === -1 ? args[index + 1] : argument.slice(equals + 1);
|
||||
if (!value || (equals === -1 && value.startsWith("-"))) throw new Error(`Missing value for ${name}.`);
|
||||
if (name === "--provider") provider = value;
|
||||
else if (name === "--model") model = value;
|
||||
else runsText = value;
|
||||
if (name !== "--runs-per-variant") cliSelectedModel = true;
|
||||
if (equals === -1) index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unsupported eval argument: ${argument}`);
|
||||
}
|
||||
|
||||
provider = provider?.trim() || undefined;
|
||||
model = model?.trim() || undefined;
|
||||
if (cliSelectedModel) {
|
||||
if (!provider || !model) throw new Error("CLI model selection requires both --provider and --model.");
|
||||
} else {
|
||||
provider = environment.PI_PROVIDER?.trim() || undefined;
|
||||
model = environment.PI_MODEL?.trim() || undefined;
|
||||
if (Boolean(provider) !== Boolean(model)) throw new Error("Set both PI_PROVIDER and PI_MODEL, or neither.");
|
||||
}
|
||||
|
||||
const configuredRuns = (runsText ?? environment.PI_EVAL_RUNS_PER_VARIANT)?.trim();
|
||||
const runsPerVariant = configuredRuns ? Number(configuredRuns) : 1;
|
||||
if (!Number.isSafeInteger(runsPerVariant) || runsPerVariant < 1) {
|
||||
throw new Error("Runs per variant must be a positive integer.");
|
||||
}
|
||||
|
||||
return { provider, model, runsPerVariant, requestedFiles, discoveryArgs };
|
||||
}
|
||||
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
function artifactRunId(): string {
|
||||
return `${new Date().toISOString().replaceAll(":", "-")}_${randomUUID()}`;
|
||||
}
|
||||
|
||||
function containerPath(path: string): string {
|
||||
const absolute = resolve(packageRoot, path);
|
||||
const packageRelative = relative(packageRoot, absolute);
|
||||
if (packageRelative.startsWith("..")) throw new Error(`Eval file must be inside ${packageRoot}: ${path}`);
|
||||
return packageRelative.replaceAll("\\", "/");
|
||||
}
|
||||
|
||||
function normalizeDiscoveredFile(path: string): string {
|
||||
const prefix = "/repo/packages/evals/";
|
||||
if (!path.startsWith(prefix)) throw new Error(`Discovered eval path is outside the container package: ${path}`);
|
||||
return path.slice(prefix.length);
|
||||
}
|
||||
|
||||
function compareDiscovery(left: readonly DiscoveredEvalCase[], right: readonly DiscoveredEvalCase[]): void {
|
||||
const identities = (cases: readonly DiscoveredEvalCase[]) =>
|
||||
cases.map(({ fullName, file }) => JSON.stringify([fullName, normalizeDiscoveredFile(file)])).sort();
|
||||
if (JSON.stringify(identities(left)) !== JSON.stringify(identities(right))) {
|
||||
throw new Error("Documentation variants discovered different eval cases.");
|
||||
}
|
||||
}
|
||||
|
||||
const cli = parseEvalCli(process.argv.slice(2));
|
||||
if (!cli.provider || !cli.model) throw new Error("Set PI_PROVIDER and PI_MODEL, or pass --provider and --model.");
|
||||
const selectedModel = { provider: cli.provider, model: cli.model };
|
||||
const modelIdentity = `${selectedModel.provider}/${selectedModel.model}`;
|
||||
const files = (
|
||||
cli.requestedFiles.length > 0 ? cli.requestedFiles : globSync("evals/**/*.docs.eval.ts", { cwd: packageRoot })
|
||||
)
|
||||
.map(containerPath)
|
||||
.sort();
|
||||
if (files.length === 0) throw new Error("No documentation eval files were selected.");
|
||||
for (const file of files) {
|
||||
if (!file.endsWith(".docs.eval.ts")) throw new Error(`Documentation runner cannot execute non-doc eval: ${file}`);
|
||||
}
|
||||
|
||||
const artifactDirectory = resolve(packageRoot, ".eval", artifactRunId());
|
||||
const authPath = requireEvalAuthFile(selectedModel.provider);
|
||||
await mkdir(artifactDirectory, { recursive: true, mode: 0o700 });
|
||||
const images = buildImages();
|
||||
if (images.with_docs.id === images.without_docs.id)
|
||||
throw new Error("Documentation variants resolved to the same image.");
|
||||
const docker = createDockerContext(
|
||||
images,
|
||||
artifactDirectory,
|
||||
authPath,
|
||||
selectedModel.provider,
|
||||
selectedModel.model,
|
||||
cli.runsPerVariant,
|
||||
);
|
||||
|
||||
const discoveries = [];
|
||||
for (const variant of DOCUMENTATION_VARIANTS) {
|
||||
const path = discoverCases(docker, variant, files, cli.discoveryArgs);
|
||||
discoveries.push(parseDiscoveredCases(JSON.parse(await readFile(path, "utf8"))));
|
||||
}
|
||||
compareDiscovery(discoveries[0], discoveries[1]);
|
||||
const cases = discoveries[0].map((evalCase) => ({ ...evalCase, file: normalizeDiscoveredFile(evalCase.file) }));
|
||||
if (cases.length === 0) throw new Error("No documentation eval cases matched the selection.");
|
||||
const tasks = createTaskPlan(cases, modelIdentity, cli.runsPerVariant);
|
||||
|
||||
const protocol = {
|
||||
schemaVersion: 1,
|
||||
model: modelIdentity,
|
||||
runsPerVariant: cli.runsPerVariant,
|
||||
images,
|
||||
files,
|
||||
cases: cases.map(({ evalSet, caseId, file }) => ({ evalSet, caseId, file })),
|
||||
tasks,
|
||||
};
|
||||
const protocolText = JSON.stringify(protocol);
|
||||
const protocolDigest = createHash("sha256").update(protocolText).digest("hex");
|
||||
await writeFile(
|
||||
resolve(artifactDirectory, "protocol.json"),
|
||||
`${JSON.stringify({ ...protocol, protocolDigest }, null, 2)}\n`,
|
||||
);
|
||||
await writeFile(resolve(artifactDirectory, "expected-runs.json"), `${JSON.stringify(tasks, null, 2)}\n`);
|
||||
|
||||
const observations: EvalObservation[] = [];
|
||||
for (const [index, task] of tasks.entries()) {
|
||||
console.log(
|
||||
`\n[${index + 1}/${tasks.length}] ${task.evalSet} > ${task.caseId} | ${task.variant} | ${task.model} | run ${task.runNumber}`,
|
||||
);
|
||||
const reportPath = runTask(docker, task);
|
||||
const observation: EvalObservation = reportPath
|
||||
? await readTaskObservation(task, reportPath, artifactDirectory)
|
||||
: erroredObservation(task);
|
||||
observations.push(observation);
|
||||
await writeFile(
|
||||
resolve(artifactDirectory, "observations.jsonl"),
|
||||
`${observations.map((item) => JSON.stringify(item)).join("\n")}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
const report = summarizeEvalObservations(protocolDigest, tasks, observations);
|
||||
const reportText = formatEvalComparisonReport(report);
|
||||
await writeFile(resolve(artifactDirectory, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
|
||||
await writeFile(resolve(artifactDirectory, "report.txt"), `${reportText}\n`);
|
||||
console.log(`\n${reportText}`);
|
||||
console.log(`\nArtifacts: ${artifactDirectory}`);
|
||||
if (report.blockedPairs.length > 0) process.exitCode = 1;
|
||||
175
third_party/pi/packages/evals/src/docker.ts
vendored
Normal file
175
third_party/pi/packages/evals/src/docker.ts
vendored
Normal file
@@ -0,0 +1,175 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, mkdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type { DocumentationVariant, EvalTask } from "./plan.ts";
|
||||
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const repositoryRoot = resolve(packageRoot, "../..");
|
||||
|
||||
export type BuiltImages = Record<DocumentationVariant, { name: string; id: string }>;
|
||||
|
||||
type DockerContext = {
|
||||
images: BuiltImages;
|
||||
artifactDirectory: string;
|
||||
authPath: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
runsPerVariant: number;
|
||||
};
|
||||
|
||||
function execute(command: string, args: readonly string[], capture = false): { status: number; stdout: string } {
|
||||
const result = spawnSync(command, args, {
|
||||
cwd: packageRoot,
|
||||
stdio: capture ? ["ignore", "pipe", "inherit"] : "inherit",
|
||||
encoding: "utf8",
|
||||
});
|
||||
if (result.error) throw result.error;
|
||||
return { status: result.status ?? 1, stdout: capture ? result.stdout : "" };
|
||||
}
|
||||
|
||||
function requireSuccess(command: string, args: readonly string[]): void {
|
||||
const { status } = execute(command, args);
|
||||
if (status !== 0) throw new Error(`${command} exited with status ${status}.`);
|
||||
}
|
||||
|
||||
export function buildImages(): BuiltImages {
|
||||
const prefix = `pi-evals-${createHash("sha256").update(repositoryRoot).digest("hex").slice(0, 12)}`;
|
||||
const images = {
|
||||
without_docs: { name: `${prefix}-without-docs:local`, id: "" },
|
||||
with_docs: { name: `${prefix}-with-docs:local`, id: "" },
|
||||
} satisfies BuiltImages;
|
||||
for (const variant of ["without_docs", "with_docs"] as const) {
|
||||
requireSuccess("docker", [
|
||||
"build",
|
||||
"--target",
|
||||
variant,
|
||||
"--tag",
|
||||
images[variant].name,
|
||||
"--file",
|
||||
join(packageRoot, "docker", "Dockerfile"),
|
||||
repositoryRoot,
|
||||
]);
|
||||
const inspected = execute("docker", ["image", "inspect", "--format", "{{.Id}}", images[variant].name], true);
|
||||
if (inspected.status !== 0 || !inspected.stdout.trim())
|
||||
throw new Error(`Cannot inspect ${images[variant].name}.`);
|
||||
images[variant].id = inspected.stdout.trim();
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
function environment(name: string, value: string): string[] {
|
||||
return ["--env", `${name}=${value}`];
|
||||
}
|
||||
|
||||
export function requireEvalAuthFile(provider: string): string {
|
||||
const path = join(
|
||||
process.env.PI_CODING_AGENT_DIR ? resolve(process.env.PI_CODING_AGENT_DIR) : join(homedir(), ".pi", "agent"),
|
||||
"auth.json",
|
||||
);
|
||||
if (!existsSync(path) || !statSync(path).isFile())
|
||||
throw new Error(`Eval authentication file does not exist: ${path}`);
|
||||
let credentials: unknown;
|
||||
try {
|
||||
credentials = JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (error) {
|
||||
throw new Error(`Eval authentication file is invalid: ${path}`, { cause: error });
|
||||
}
|
||||
if (
|
||||
typeof credentials !== "object" ||
|
||||
credentials === null ||
|
||||
Array.isArray(credentials) ||
|
||||
!Object.hasOwn(credentials, provider)
|
||||
) {
|
||||
throw new Error(`Eval authentication file has no credential for provider ${provider}.`);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
function dockerArgs(
|
||||
context: DockerContext,
|
||||
variant: DocumentationVariant,
|
||||
outputDirectory: string,
|
||||
entrypointArgs: readonly string[],
|
||||
): string[] {
|
||||
mkdirSync(outputDirectory, { recursive: true, mode: 0o700 });
|
||||
const args = [
|
||||
"run",
|
||||
"--rm",
|
||||
"--read-only",
|
||||
"--tmpfs",
|
||||
"/tmp:rw,exec,mode=1777",
|
||||
"--tmpfs",
|
||||
"/repo/node_modules/.vite-temp:rw,exec,mode=1777",
|
||||
"--mount",
|
||||
`type=bind,source=${outputDirectory},target=/artifacts`,
|
||||
...environment("PI_EVAL_ARTIFACT_DIR", "/artifacts"),
|
||||
...environment("PI_EVAL_RUNS_PER_VARIANT", String(context.runsPerVariant)),
|
||||
...environment("PI_EVAL_SANDBOX_UID", "65532"),
|
||||
...environment("PI_EVAL_SANDBOX_GID", "65532"),
|
||||
...environment("PI_PROVIDER", context.provider),
|
||||
...environment("PI_MODEL", context.model),
|
||||
];
|
||||
if (typeof process.getuid === "function" && typeof process.getgid === "function") {
|
||||
args.push(
|
||||
...environment("PI_EVAL_ARTIFACT_UID", String(process.getuid())),
|
||||
...environment("PI_EVAL_ARTIFACT_GID", String(process.getgid())),
|
||||
);
|
||||
}
|
||||
args.push("--mount", `type=bind,source=${context.authPath},target=/run/pi-eval-secrets/auth.json,readonly`);
|
||||
args.push(context.images[variant].name, ...entrypointArgs);
|
||||
return args;
|
||||
}
|
||||
|
||||
export function createDockerContext(
|
||||
images: BuiltImages,
|
||||
artifactDirectory: string,
|
||||
authPath: string,
|
||||
provider: string,
|
||||
model: string,
|
||||
runsPerVariant: number,
|
||||
): DockerContext {
|
||||
return { images, artifactDirectory, authPath, provider, model, runsPerVariant };
|
||||
}
|
||||
|
||||
export function discoverCases(
|
||||
context: DockerContext,
|
||||
variant: DocumentationVariant,
|
||||
files: readonly string[],
|
||||
vitestArgs: readonly string[],
|
||||
): string {
|
||||
const outputDirectory = join(context.artifactDirectory, "discovery", variant);
|
||||
const args = dockerArgs(context, variant, outputDirectory, [
|
||||
"--discover",
|
||||
"--project",
|
||||
"docs",
|
||||
...files,
|
||||
...vitestArgs,
|
||||
]);
|
||||
const { status } = execute("docker", args);
|
||||
if (status !== 0) throw new Error(`${variant} eval discovery failed.`);
|
||||
return join(outputDirectory, "discovered-tests.json");
|
||||
}
|
||||
|
||||
function taskDirectoryName(task: EvalTask): string {
|
||||
const identity = JSON.stringify([task.evalSet, task.caseId, task.variant, task.model, task.runNumber]);
|
||||
return createHash("sha256").update(identity).digest("hex");
|
||||
}
|
||||
|
||||
export function runTask(context: DockerContext, task: EvalTask): string | undefined {
|
||||
const outputDirectory = join(context.artifactDirectory, "tasks", taskDirectoryName(task));
|
||||
const testName = `${task.evalSet} ${task.caseId}`;
|
||||
const exactName = `^${testName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}$`;
|
||||
const args = dockerArgs(context, task.variant, outputDirectory, [
|
||||
"--project",
|
||||
"docs",
|
||||
task.file,
|
||||
"--testNamePattern",
|
||||
exactName,
|
||||
]);
|
||||
execute("docker", args);
|
||||
const reportPath = join(outputDirectory, "vitest.json");
|
||||
return existsSync(reportPath) ? reportPath : undefined;
|
||||
}
|
||||
140
third_party/pi/packages/evals/src/extensions.eval.ts
vendored
140
third_party/pi/packages/evals/src/extensions.eval.ts
vendored
@@ -1,140 +0,0 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect } from "vitest";
|
||||
import { createJudge, describeEval } from "vitest-evals";
|
||||
import { createPiCodingAgentHarness, type PiCodingAgentInput } from "./pi-harness.ts";
|
||||
import { recordEvalSourceArtifact } from "./vitest-evals/artifacts.ts";
|
||||
import { evalHarnessTable } from "./vitest-evals/harness-table.ts";
|
||||
|
||||
type ExtensionAuthoringOutput = {
|
||||
response: string;
|
||||
systemPromptHasGuidelines: boolean;
|
||||
systemPromptHasPiDocs: boolean;
|
||||
extensionErrors: Array<{ path: string; error: string }>;
|
||||
loadedExtensions: Array<{ path: string; tools: string[] }>;
|
||||
extensionSource: string | null;
|
||||
};
|
||||
|
||||
function createExtensionAuthoringHarness(name: string, transformSystemPrompt?: (defaultPrompt: string) => string) {
|
||||
return createPiCodingAgentHarness({
|
||||
name,
|
||||
...(transformSystemPrompt ? { transformSystemPrompt } : {}),
|
||||
output: ({ response, session }) => {
|
||||
const extensions = session.resourceLoader.getExtensions();
|
||||
const extensionPath = join(session.sessionManager.getCwd(), ".pi", "extensions", "hello.ts");
|
||||
const extensionSource = existsSync(extensionPath) ? readFileSync(extensionPath, "utf8") : null;
|
||||
return {
|
||||
response,
|
||||
systemPromptHasGuidelines: session.systemPrompt.includes("\nGuidelines:\n"),
|
||||
systemPromptHasPiDocs: session.systemPrompt.includes("\nPi documentation (read only"),
|
||||
extensionErrors: extensions.errors,
|
||||
loadedExtensions: extensions.extensions.map(({ path, tools }) => ({
|
||||
path,
|
||||
tools: [...tools.keys()],
|
||||
})),
|
||||
extensionSource,
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function excludeGuidelinesAndDocumentation(defaultPrompt: string): string {
|
||||
const guidelinesStart = defaultPrompt.indexOf("\nGuidelines:\n");
|
||||
if (guidelinesStart === -1) throw new Error("Default Pi system prompt has no Guidelines section.");
|
||||
return defaultPrompt.slice(0, guidelinesStart);
|
||||
}
|
||||
|
||||
function prepareDefaultPromptOverride(defaultPrompt: string): string {
|
||||
const cwdStart = defaultPrompt.lastIndexOf("\nCurrent working directory: ");
|
||||
if (cwdStart === -1) throw new Error("Default Pi system prompt has no working-directory section.");
|
||||
return defaultPrompt.slice(0, cwdStart);
|
||||
}
|
||||
|
||||
const ExtensionAuthoringJudge = createJudge<PiCodingAgentInput, ExtensionAuthoringOutput>(
|
||||
"ExtensionAuthoringJudge",
|
||||
({ output, toolCalls }) => {
|
||||
const failures: string[] = [];
|
||||
if (output.extensionSource === null) {
|
||||
failures.push("generated extension source is unavailable");
|
||||
} else {
|
||||
const imports = Array.from(
|
||||
output.extensionSource.matchAll(/\b(?:from|import)\s+["']([^"']+)["']/g),
|
||||
(match) => match[1],
|
||||
);
|
||||
if (!imports.includes("@earendil-works/pi-coding-agent")) {
|
||||
failures.push("extension does not import the canonical @earendil-works/pi-coding-agent package");
|
||||
}
|
||||
if (imports.some((specifier) => specifier.startsWith("@mariozechner/"))) {
|
||||
failures.push("extension imports a legacy @mariozechner package");
|
||||
}
|
||||
if (imports.some((specifier) => specifier.startsWith("@sinclair/typebox"))) {
|
||||
failures.push('extension imports legacy "@sinclair/typebox" instead of "typebox"');
|
||||
}
|
||||
}
|
||||
if (output.extensionErrors.length > 0) failures.push("extension loader reported errors");
|
||||
if (!output.loadedExtensions.some(({ tools }) => tools.includes("hello"))) {
|
||||
failures.push('no loaded extension registered the "hello" tool');
|
||||
}
|
||||
if (
|
||||
!toolCalls.some(
|
||||
(call) =>
|
||||
call.name === "hello" &&
|
||||
call.status === "ok" &&
|
||||
call.arguments?.name === "Bob" &&
|
||||
call.result === "Hello, Bob!",
|
||||
)
|
||||
) {
|
||||
failures.push('no successful hello({ name: "Bob" }) call returned "Hello, Bob!"');
|
||||
}
|
||||
if (output.response !== "Hello, Bob!") failures.push('final response was not exactly "Hello, Bob!"');
|
||||
|
||||
return {
|
||||
score: failures.length === 0 ? 1 : 0,
|
||||
metadata: {
|
||||
rationale: failures.length === 0 ? "Extension authoring workflow completed." : failures.join("; "),
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const extensionHarnessTable = evalHarnessTable("Pi extension authoring system prompt", {
|
||||
baseline: createExtensionAuthoringHarness("system-prompt-without-docs", excludeGuidelinesAndDocumentation),
|
||||
candidate: createExtensionAuthoringHarness("default-system-prompt", prepareDefaultPromptOverride),
|
||||
});
|
||||
|
||||
describe.for(extensionHarnessTable)("$name", ({ harness }) => {
|
||||
describeEval(
|
||||
"Pi extension authoring system prompt",
|
||||
{ harness, judges: [ExtensionAuthoringJudge], judgeThreshold: null },
|
||||
(it) => {
|
||||
it("creates, reloads, and uses a hello extension", async ({ run, task }) => {
|
||||
const result = await run([
|
||||
{
|
||||
type: "prompt",
|
||||
content:
|
||||
"Create a Pi extension with a hello tool that takes a name and returns a greeting. For example, passing Bob should return `Hello, Bob!`.",
|
||||
},
|
||||
{ type: "reload" },
|
||||
{
|
||||
type: "prompt",
|
||||
content:
|
||||
"Use the hello tool to greet Bob. Respond with exactly the tool's greeting and nothing else.",
|
||||
},
|
||||
]);
|
||||
if (result.output.extensionSource !== null) {
|
||||
const runId = result.artifacts?.runId;
|
||||
if (typeof runId !== "string") throw new Error("Pi eval run did not record a run ID.");
|
||||
await recordEvalSourceArtifact(task, runId, {
|
||||
name: "hello.ts",
|
||||
contentType: "text/typescript",
|
||||
body: result.output.extensionSource,
|
||||
bodyEncoding: "utf-8",
|
||||
});
|
||||
}
|
||||
const expectsFullPrompt = harness.name === "default-system-prompt";
|
||||
expect(result.output.systemPromptHasGuidelines).toBe(expectsFullPrompt);
|
||||
expect(result.output.systemPromptHasPiDocs).toBe(expectsFullPrompt);
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
537
third_party/pi/packages/evals/src/harness.ts
vendored
Normal file
537
third_party/pi/packages/evals/src/harness.ts
vendored
Normal file
@@ -0,0 +1,537 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { Dirent } from "node:fs";
|
||||
import { existsSync } from "node:fs";
|
||||
import { chmod, chown, lstat, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { contentText, InMemoryCredentialStore } from "@earendil-works/pi-ai";
|
||||
import { getCurrentSystemPrompt } from "@earendil-works/pi-ai/utils/transcript";
|
||||
import {
|
||||
type AgentSession,
|
||||
type CreateAgentSessionOptions,
|
||||
createAgentSessionFromServices,
|
||||
createAgentSessionServices,
|
||||
getAgentDir,
|
||||
type InlineExtension,
|
||||
ModelRuntime,
|
||||
readStoredCredential,
|
||||
SessionManager,
|
||||
} from "@earendil-works/pi-coding-agent";
|
||||
import {
|
||||
attachHarnessRunToError,
|
||||
createHarness,
|
||||
type Harness,
|
||||
type HarnessContext,
|
||||
type JsonValue,
|
||||
normalizeHarnessRun,
|
||||
normalizeRecord,
|
||||
type SimpleHarnessResult,
|
||||
type TranscriptEvent,
|
||||
toJsonValue,
|
||||
type UsageSummary,
|
||||
} from "vitest-evals/harness";
|
||||
import type { DocumentationVariant } from "./plan.ts";
|
||||
import { PI_SESSION_SNAPSHOT_ARTIFACT } from "./report.ts";
|
||||
|
||||
type PiRunDiagnostics = {
|
||||
events: TranscriptEvent[];
|
||||
metadata: Record<string, unknown>;
|
||||
usage: UsageSummary;
|
||||
};
|
||||
|
||||
export type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>;
|
||||
|
||||
export type PiCodingAgentModelSelection = {
|
||||
provider: string;
|
||||
id: string;
|
||||
};
|
||||
|
||||
export type PiCodingAgentHarnessOptions = {
|
||||
name?: string;
|
||||
model?: PiCodingAgentModelSelection;
|
||||
noTools?: CreateAgentSessionOptions["noTools"];
|
||||
tools?: CreateAgentSessionOptions["tools"];
|
||||
customTools?: CreateAgentSessionOptions["customTools"];
|
||||
workspaceFiles?: Readonly<Record<string, string>>;
|
||||
transformSystemPrompt?: (defaultPrompt: string) => string;
|
||||
expectedPiDocumentation?: boolean;
|
||||
};
|
||||
|
||||
export type PiCodingAgentHarnessWithOutput<TOutput extends JsonValue> = PiCodingAgentHarnessOptions & {
|
||||
output: (args: {
|
||||
response: string;
|
||||
session: AgentSession;
|
||||
systemPrompt: string;
|
||||
agentDir: string;
|
||||
}) => TOutput | Promise<TOutput>;
|
||||
};
|
||||
|
||||
export function resolveModelSelection(
|
||||
explicitModel: PiCodingAgentModelSelection | undefined,
|
||||
environment: { PI_PROVIDER?: string; PI_MODEL?: string } = process.env,
|
||||
): PiCodingAgentModelSelection {
|
||||
const provider = (explicitModel?.provider ?? environment.PI_PROVIDER)?.trim();
|
||||
const id = (explicitModel?.id ?? environment.PI_MODEL)?.trim();
|
||||
if (!provider || !id) {
|
||||
throw new Error("Select a harness model explicitly or set both PI_PROVIDER and PI_MODEL as defaults.");
|
||||
}
|
||||
return { provider, id };
|
||||
}
|
||||
|
||||
export function applyIsolatedEnvironment(home: string, agentDir: string): () => void {
|
||||
const overrides = { HOME: home, USERPROFILE: home, PI_CODING_AGENT_DIR: agentDir };
|
||||
const previous = new Map<string, string | undefined>();
|
||||
for (const name of Object.keys(process.env)) {
|
||||
if (!name.startsWith("PI_EVAL_")) continue;
|
||||
previous.set(name, process.env[name]);
|
||||
delete process.env[name];
|
||||
}
|
||||
for (const [name, value] of Object.entries(overrides)) {
|
||||
if (!previous.has(name)) previous.set(name, process.env[name]);
|
||||
process.env[name] = value;
|
||||
}
|
||||
return () => {
|
||||
for (const [name, value] of previous) {
|
||||
if (value === undefined) delete process.env[name];
|
||||
else process.env[name] = value;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
type SandboxIdentity = { uid: number; gid: number };
|
||||
|
||||
function parseSandboxId(name: "PI_EVAL_SANDBOX_UID" | "PI_EVAL_SANDBOX_GID"): number | undefined {
|
||||
const value = process.env[name];
|
||||
if (value === undefined) return undefined;
|
||||
const id = Number(value);
|
||||
if (!Number.isSafeInteger(id) || id < 1) throw new Error(`${name} must be a positive integer.`);
|
||||
return id;
|
||||
}
|
||||
|
||||
function resolveSandboxIdentity(): SandboxIdentity | undefined {
|
||||
const uid = parseSandboxId("PI_EVAL_SANDBOX_UID");
|
||||
const gid = parseSandboxId("PI_EVAL_SANDBOX_GID");
|
||||
if (uid === undefined && gid === undefined) return undefined;
|
||||
if (uid === undefined || gid === undefined) {
|
||||
throw new Error("Set both PI_EVAL_SANDBOX_UID and PI_EVAL_SANDBOX_GID, or neither.");
|
||||
}
|
||||
return { uid, gid };
|
||||
}
|
||||
|
||||
async function chownTree(path: string, identity: SandboxIdentity): Promise<void> {
|
||||
const stats = await lstat(path);
|
||||
if (stats.isDirectory() && !stats.isSymbolicLink()) {
|
||||
await Promise.all((await readdir(path)).map((entry) => chownTree(join(path, entry), identity)));
|
||||
}
|
||||
await chown(path, identity.uid, identity.gid);
|
||||
}
|
||||
|
||||
async function protectTransformedModules(): Promise<string[]> {
|
||||
const protectedPaths: string[] = [];
|
||||
for (const directory of await readdir(tmpdir(), { withFileTypes: true })) {
|
||||
if (!directory.isDirectory()) continue;
|
||||
const cacheRoot = join(tmpdir(), directory.name);
|
||||
const ssrRoot = join(cacheRoot, "ssr");
|
||||
let files: Dirent[];
|
||||
try {
|
||||
files = await readdir(ssrRoot, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if (error instanceof Error && "code" in error && error.code === "ENOENT") continue;
|
||||
throw error;
|
||||
}
|
||||
await Promise.all([chmod(cacheRoot, 0o755), chmod(ssrRoot, 0o755)]);
|
||||
for (const file of files) {
|
||||
if (!file.isFile()) continue;
|
||||
const path = join(ssrRoot, file.name);
|
||||
await chmod(path, 0o600);
|
||||
protectedPaths.push(path);
|
||||
}
|
||||
}
|
||||
return protectedPaths;
|
||||
}
|
||||
|
||||
async function enterToolSandbox(root: string, identity: SandboxIdentity | undefined): Promise<void> {
|
||||
if (!identity) return;
|
||||
if (
|
||||
typeof process.getuid !== "function" ||
|
||||
typeof process.geteuid !== "function" ||
|
||||
typeof process.setuid !== "function" ||
|
||||
typeof process.setgid !== "function" ||
|
||||
typeof process.setgroups !== "function"
|
||||
) {
|
||||
throw new Error("The eval filesystem sandbox requires POSIX user APIs.");
|
||||
}
|
||||
if (process.getuid() !== 0 || process.geteuid() !== 0) {
|
||||
throw new Error("The eval runner must start as root before entering the unprivileged tool sandbox.");
|
||||
}
|
||||
|
||||
const protectedPaths = await protectTransformedModules();
|
||||
await chownTree(root, identity);
|
||||
process.setgroups([]);
|
||||
process.setgid(identity.gid);
|
||||
process.setuid(identity.uid);
|
||||
if (process.getuid() !== identity.uid || process.geteuid() !== identity.uid) {
|
||||
throw new Error("Failed to enter the unprivileged eval tool sandbox.");
|
||||
}
|
||||
for (const path of protectedPaths) {
|
||||
try {
|
||||
await readFile(path);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && "code" in error && (error.code === "EACCES" || error.code === "ENOENT")) {
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
throw new Error(`The unprivileged eval process can read a transformed eval module: ${path}`);
|
||||
}
|
||||
}
|
||||
|
||||
function toTranscriptEvents(messages: AgentSession["messages"]): TranscriptEvent[] {
|
||||
const events: TranscriptEvent[] = [];
|
||||
for (const message of messages) {
|
||||
if (message.role === "user") {
|
||||
events.push({ type: "message", role: "user", content: contentText(message.content) });
|
||||
continue;
|
||||
}
|
||||
if (message.role === "assistant") {
|
||||
const text = contentText(message.content);
|
||||
if (text) events.push({ type: "message", role: "assistant", content: text });
|
||||
for (const part of message.content) {
|
||||
if (part.type !== "toolCall") continue;
|
||||
events.push({
|
||||
type: "tool_call",
|
||||
id: part.id,
|
||||
name: part.name,
|
||||
arguments: normalizeRecord(part.arguments),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (message.role === "toolResult") {
|
||||
const text = contentText(message.content);
|
||||
events.push({
|
||||
type: "tool_result",
|
||||
toolCallId: message.toolCallId,
|
||||
name: message.toolName,
|
||||
content: message.content.every((part) => part.type === "text") ? text : toJsonValue(message.content),
|
||||
...(message.isError ? { error: { message: text || "Tool failed" } } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
async function seedWorkspace(workspace: string, files: Readonly<Record<string, string>> | undefined): Promise<void> {
|
||||
for (const [name, content] of Object.entries(files ?? {})) {
|
||||
if (!name || isAbsolute(name)) throw new TypeError(`Invalid workspace fixture path: ${name}`);
|
||||
const path = resolve(workspace, name);
|
||||
const pathFromWorkspace = relative(workspace, path);
|
||||
if (pathFromWorkspace === ".." || pathFromWorkspace.startsWith(`..${sep}`) || isAbsolute(pathFromWorkspace)) {
|
||||
throw new TypeError(`Workspace fixture escapes the workspace: ${name}`);
|
||||
}
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
await writeFile(path, content, { mode: 0o600 });
|
||||
}
|
||||
}
|
||||
|
||||
async function promptAgent(session: AgentSession, input: string, signal: AbortSignal | undefined): Promise<string> {
|
||||
signal?.throwIfAborted();
|
||||
const previousMessageCount = session.messages.length;
|
||||
await session.prompt(input);
|
||||
const assistant = session.messages
|
||||
.slice(previousMessageCount)
|
||||
.reverse()
|
||||
.find((message) => message.role === "assistant");
|
||||
if (!assistant) throw new Error("Agent run completed without an assistant message.");
|
||||
if (assistant.stopReason !== "stop" && assistant.stopReason !== "toolUse") {
|
||||
throw new Error(
|
||||
assistant.errorMessage ?? `Agent run ended with unexpected stop reason: ${assistant.stopReason}.`,
|
||||
);
|
||||
}
|
||||
const output = session.getLastAssistantText();
|
||||
if (!output && assistant.stopReason === "stop") throw new Error("Agent run produced no assistant text.");
|
||||
return output ?? "";
|
||||
}
|
||||
|
||||
export function verifySystemPrompt(
|
||||
systemPrompt: string,
|
||||
options: Pick<PiCodingAgentHarnessOptions, "name" | "expectedPiDocumentation">,
|
||||
): string {
|
||||
if (options.expectedPiDocumentation === undefined) return systemPrompt;
|
||||
if (!systemPrompt.includes("\n<rules>\n")) {
|
||||
throw new Error(`Pi system prompt lost its rules in the ${options.name} eval variant.`);
|
||||
}
|
||||
const hasDocumentation = systemPrompt.includes("\n<docs>\nPi documentation (read only");
|
||||
if (hasDocumentation !== options.expectedPiDocumentation) {
|
||||
throw new Error(`Pi system prompt does not match the ${options.name} eval variant.`);
|
||||
}
|
||||
return systemPrompt;
|
||||
}
|
||||
|
||||
async function runPiCodingAgent<TOutput extends JsonValue>(
|
||||
input: PiCodingAgentInput,
|
||||
signal: AbortSignal | undefined,
|
||||
setArtifact: HarnessContext["setArtifact"],
|
||||
options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput<TOutput>,
|
||||
): Promise<SimpleHarnessResult<string | TOutput>> {
|
||||
const startedAt = performance.now();
|
||||
signal?.throwIfAborted();
|
||||
const selection = resolveModelSelection(options.model);
|
||||
const hostAgentDir = getAgentDir();
|
||||
const sandboxIdentity = resolveSandboxIdentity();
|
||||
const root = await mkdtemp(join(tmpdir(), "pi-eval-"));
|
||||
const workspace = join(root, "workspace");
|
||||
const isolatedHome = join(root, "home");
|
||||
const agentDir = join(isolatedHome, ".pi", "agent");
|
||||
const extensionFactories: InlineExtension[] = [];
|
||||
let forcedSystemPrompt: string | undefined;
|
||||
if (options.transformSystemPrompt) {
|
||||
const transform = options.transformSystemPrompt;
|
||||
extensionFactories.push({
|
||||
name: "eval-system-prompt-transform",
|
||||
hidden: true,
|
||||
factory: (pi) => {
|
||||
pi.on("before_agent_start", ({ systemPrompt }) => {
|
||||
forcedSystemPrompt = transform(systemPrompt);
|
||||
return { systemPrompt: forcedSystemPrompt };
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
let sessionManager: SessionManager | undefined;
|
||||
let session: AgentSession | undefined;
|
||||
let result: SimpleHarnessResult<string | TOutput> | undefined;
|
||||
let runDiagnostics: PiRunDiagnostics | undefined;
|
||||
let runError: unknown;
|
||||
const cleanupErrors: unknown[] = [];
|
||||
let hiddenCredentialEnvironment: { name: string; value: string } | undefined;
|
||||
const restoreEnvironment = applyIsolatedEnvironment(isolatedHome, agentDir);
|
||||
try {
|
||||
const authPath = join(hostAgentDir, "auth.json");
|
||||
const credentials = new InMemoryCredentialStore();
|
||||
const storedCredential = readStoredCredential(selection.provider, authPath);
|
||||
if (storedCredential) await credentials.modify(selection.provider, async () => storedCredential);
|
||||
const modelRuntime = await ModelRuntime.create({ credentials });
|
||||
await Promise.all([mkdir(workspace), mkdir(agentDir, { recursive: true })]);
|
||||
await seedWorkspace(workspace, options.workspaceFiles);
|
||||
const model = modelRuntime.getModel(selection.provider, selection.id);
|
||||
if (!model) throw new Error(`Eval model not found: ${selection.provider}/${selection.id}`);
|
||||
const auth = await modelRuntime.getAuth(model);
|
||||
if (!auth) {
|
||||
throw new Error(`Eval model has no configured authentication: ${selection.provider}/${selection.id}`);
|
||||
}
|
||||
if (!storedCredential && auth.auth.apiKey) {
|
||||
await modelRuntime.setRuntimeApiKey(selection.provider, auth.auth.apiKey);
|
||||
}
|
||||
if (sandboxIdentity) {
|
||||
await rm(authPath, { force: true });
|
||||
const credentialEnvironmentValue = auth.source ? process.env[auth.source] : undefined;
|
||||
if (auth.source && /^[A-Z][A-Z0-9_]*$/.test(auth.source) && credentialEnvironmentValue) {
|
||||
hiddenCredentialEnvironment = { name: auth.source, value: credentialEnvironmentValue };
|
||||
delete process.env[auth.source];
|
||||
}
|
||||
}
|
||||
|
||||
const services = await createAgentSessionServices({
|
||||
cwd: workspace,
|
||||
modelRuntime,
|
||||
resourceLoaderOptions: { extensionFactories },
|
||||
});
|
||||
signal?.throwIfAborted();
|
||||
sessionManager = SessionManager.create(workspace, join(root, "sessions"));
|
||||
setArtifact("runId", sessionManager.getSessionId());
|
||||
session = (
|
||||
await createAgentSessionFromServices({
|
||||
services,
|
||||
sessionManager,
|
||||
model,
|
||||
thinkingLevel: "off",
|
||||
tools: options.tools,
|
||||
noTools: options.noTools,
|
||||
customTools: options.customTools,
|
||||
})
|
||||
).session;
|
||||
|
||||
const expectedInlinePaths = new Set(extensionFactories.map(({ name }) => `<inline:${name}>`));
|
||||
const unexpectedExtensions = session.extensionRunner
|
||||
.getExtensionPaths()
|
||||
.filter((path) => !expectedInlinePaths.has(path));
|
||||
if (unexpectedExtensions.length > 0) {
|
||||
throw new Error(`Isolated eval loaded unexpected extensions: ${unexpectedExtensions.join(", ")}`);
|
||||
}
|
||||
|
||||
await enterToolSandbox(root, sandboxIdentity);
|
||||
let response: string | undefined;
|
||||
const steps = typeof input === "string" ? [{ type: "prompt" as const, content: input }] : input;
|
||||
let abortPromise: Promise<void> | undefined;
|
||||
const abort = () => {
|
||||
abortPromise ??= session!.abort();
|
||||
};
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
try {
|
||||
for (const step of steps) {
|
||||
if (step.type === "reload") {
|
||||
await session.reload();
|
||||
continue;
|
||||
}
|
||||
response = await promptAgent(session, step.content, signal);
|
||||
}
|
||||
} finally {
|
||||
signal?.removeEventListener("abort", abort);
|
||||
if (abortPromise) await abortPromise;
|
||||
}
|
||||
if (response === undefined) {
|
||||
throw new Error("Pi eval input must include at least one prompt step.");
|
||||
}
|
||||
// A forced prompt is not recorded in the transcript, so use the one the transform
|
||||
// extension sent; otherwise the replayed transcript prompt is what the provider received.
|
||||
const systemPrompt = forcedSystemPrompt ?? getCurrentSystemPrompt(session.messages);
|
||||
const stats = session.getSessionStats();
|
||||
const hasPricing = [model.cost, ...(model.cost.tiers ?? [])].some(
|
||||
({ input: inputCost, output: outputCost, cacheRead, cacheWrite }) =>
|
||||
inputCost > 0 || outputCost > 0 || cacheRead > 0 || cacheWrite > 0,
|
||||
);
|
||||
runDiagnostics = {
|
||||
events: toTranscriptEvents(session.messages),
|
||||
metadata: { systemPromptSha256: createHash("sha256").update(systemPrompt).digest("hex") },
|
||||
usage: {
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
inputTokens: stats.tokens.input,
|
||||
outputTokens: stats.tokens.output,
|
||||
totalTokens: stats.tokens.total,
|
||||
toolCalls: stats.toolCalls,
|
||||
metadata: {
|
||||
cacheReadTokens: stats.tokens.cacheRead,
|
||||
cacheWriteTokens: stats.tokens.cacheWrite,
|
||||
...(hasPricing ? { estimatedCostUsd: stats.cost } : {}),
|
||||
},
|
||||
},
|
||||
};
|
||||
verifySystemPrompt(systemPrompt, options);
|
||||
const output =
|
||||
"output" in options ? await options.output({ response, session, systemPrompt, agentDir }) : response;
|
||||
result = { output, ...runDiagnostics };
|
||||
} catch (error) {
|
||||
runError = error;
|
||||
} finally {
|
||||
if (sessionManager) {
|
||||
const sessionPath = sessionManager.getSessionFile();
|
||||
if (!sessionPath || !existsSync(sessionPath)) {
|
||||
cleanupErrors.push(new Error("Pi eval produced no session file."));
|
||||
} else {
|
||||
try {
|
||||
setArtifact(PI_SESSION_SNAPSHOT_ARTIFACT, await readFile(sessionPath, "utf8"));
|
||||
} catch (error) {
|
||||
cleanupErrors.push(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
session?.dispose();
|
||||
} catch (error) {
|
||||
cleanupErrors.push(error);
|
||||
}
|
||||
try {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
} catch (error) {
|
||||
cleanupErrors.push(error);
|
||||
}
|
||||
restoreEnvironment();
|
||||
if (hiddenCredentialEnvironment) {
|
||||
process.env[hiddenCredentialEnvironment.name] = hiddenCredentialEnvironment.value;
|
||||
}
|
||||
}
|
||||
|
||||
let failure = runError;
|
||||
if (runError !== undefined && cleanupErrors.length > 0) {
|
||||
failure = new AggregateError([runError, ...cleanupErrors], "Agent run failed and cleanup also failed.");
|
||||
} else if (cleanupErrors.length === 1) {
|
||||
failure = cleanupErrors[0];
|
||||
} else if (cleanupErrors.length > 1) {
|
||||
failure = new AggregateError(cleanupErrors, "Agent cleanup failed.");
|
||||
}
|
||||
if (failure !== undefined) {
|
||||
if (runDiagnostics) {
|
||||
const partialRun = normalizeHarnessRun(input, {
|
||||
...runDiagnostics,
|
||||
errors: [failure],
|
||||
timings: { totalMs: performance.now() - startedAt },
|
||||
});
|
||||
throw attachHarnessRunToError(failure, partialRun);
|
||||
}
|
||||
throw failure;
|
||||
}
|
||||
if (!result) throw new Error("Pi eval completed without a result.");
|
||||
return { ...result, timings: { totalMs: performance.now() - startedAt } };
|
||||
}
|
||||
|
||||
export function createPiCodingAgentHarness<TOutput extends JsonValue>(
|
||||
options: PiCodingAgentHarnessWithOutput<TOutput>,
|
||||
): Harness<PiCodingAgentInput, TOutput>;
|
||||
export function createPiCodingAgentHarness(options?: PiCodingAgentHarnessOptions): Harness<PiCodingAgentInput, string>;
|
||||
export function createPiCodingAgentHarness<TOutput extends JsonValue>(
|
||||
options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput<TOutput> = {},
|
||||
): Harness<PiCodingAgentInput, string | TOutput> {
|
||||
return createHarness<PiCodingAgentInput, string | TOutput>({
|
||||
name: options.name ?? "pi-coding-agent",
|
||||
run: ({ input, signal, setArtifact }) => runPiCodingAgent(input, signal, setArtifact, options),
|
||||
});
|
||||
}
|
||||
|
||||
/** Documentation evals intentionally exclude shell and unrestricted network tools. */
|
||||
export const DOCUMENTATION_EVAL_TOOLS = ["read", "write", "edit", "grep", "find", "ls"] as const;
|
||||
|
||||
export function resolveDocumentationVariant(
|
||||
value: string | undefined = process.env.PI_EVAL_VARIANT,
|
||||
): DocumentationVariant {
|
||||
if (value === "without_docs" || value === "with_docs") return value;
|
||||
throw new TypeError('PI_EVAL_VARIANT must be "without_docs" or "with_docs".');
|
||||
}
|
||||
|
||||
export function excludePiDocumentation(defaultPrompt: string): string {
|
||||
const documentationStartMarker = "\n<docs>\n";
|
||||
const documentationEndMarker = "\n</docs>";
|
||||
const documentationStart = defaultPrompt.indexOf(documentationStartMarker);
|
||||
if (documentationStart === -1) throw new Error("Default Pi system prompt has no Pi documentation section.");
|
||||
const documentationEnd = defaultPrompt.indexOf(documentationEndMarker, documentationStart);
|
||||
if (documentationEnd === -1) throw new Error("Default Pi system prompt has no complete Pi documentation section.");
|
||||
const cwdStart = defaultPrompt.lastIndexOf("\n<cwd>\n");
|
||||
if (cwdStart < documentationEnd) throw new Error("Default Pi system prompt has no working-directory section.");
|
||||
return (
|
||||
defaultPrompt.slice(0, documentationStart) + defaultPrompt.slice(documentationEnd + documentationEndMarker.length)
|
||||
);
|
||||
}
|
||||
|
||||
type DocumentationHarnessOptions = Omit<
|
||||
PiCodingAgentHarnessOptions,
|
||||
"name" | "transformSystemPrompt" | "expectedPiDocumentation"
|
||||
>;
|
||||
type DocumentationHarnessWithOutput<TOutput extends JsonValue> = Omit<
|
||||
PiCodingAgentHarnessWithOutput<TOutput>,
|
||||
"name" | "transformSystemPrompt" | "expectedPiDocumentation"
|
||||
>;
|
||||
|
||||
export function createPiDocumentationEvalHarness<TOutput extends JsonValue>(
|
||||
options: DocumentationHarnessWithOutput<TOutput>,
|
||||
): Harness<PiCodingAgentInput, TOutput>;
|
||||
export function createPiDocumentationEvalHarness(
|
||||
options?: DocumentationHarnessOptions,
|
||||
): Harness<PiCodingAgentInput, string>;
|
||||
export function createPiDocumentationEvalHarness<TOutput extends JsonValue>(
|
||||
options: DocumentationHarnessOptions | DocumentationHarnessWithOutput<TOutput> = {},
|
||||
) {
|
||||
if (process.env.PI_EVAL_CONTAINER !== "1" || !resolveSandboxIdentity()) {
|
||||
throw new Error("Documentation evals must run in the isolated container sandbox.");
|
||||
}
|
||||
const variant = resolveDocumentationVariant();
|
||||
return createPiCodingAgentHarness({
|
||||
...options,
|
||||
name: variant,
|
||||
tools: options.tools ?? [...DOCUMENTATION_EVAL_TOOLS],
|
||||
...(variant === "without_docs" ? { transformSystemPrompt: excludePiDocumentation } : {}),
|
||||
expectedPiDocumentation: variant === "with_docs",
|
||||
});
|
||||
}
|
||||
257
third_party/pi/packages/evals/src/pi-harness.ts
vendored
257
third_party/pi/packages/evals/src/pi-harness.ts
vendored
@@ -1,257 +0,0 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { contentText } from "@earendil-works/pi-ai";
|
||||
import {
|
||||
type AgentSession,
|
||||
type CreateAgentSessionOptions,
|
||||
createAgentSessionFromServices,
|
||||
createAgentSessionServices,
|
||||
ModelRuntime,
|
||||
SessionManager,
|
||||
SettingsManager,
|
||||
} from "@earendil-works/pi-coding-agent";
|
||||
import {
|
||||
createHarness,
|
||||
type Harness,
|
||||
type HarnessContext,
|
||||
type JsonValue,
|
||||
normalizeRecord,
|
||||
type SimpleHarnessResult,
|
||||
type TranscriptEvent,
|
||||
toJsonValue,
|
||||
} from "vitest-evals/harness";
|
||||
import { PI_SESSION_SNAPSHOT_ARTIFACT } from "./vitest-evals/artifacts.ts";
|
||||
|
||||
export type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>;
|
||||
|
||||
type PiCodingAgentModelSelection = {
|
||||
provider: string;
|
||||
id: string;
|
||||
};
|
||||
|
||||
type PiCodingAgentHarnessOptions = {
|
||||
name?: string;
|
||||
model?: PiCodingAgentModelSelection;
|
||||
noTools?: CreateAgentSessionOptions["noTools"];
|
||||
transformSystemPrompt?: (defaultPrompt: string) => string;
|
||||
};
|
||||
|
||||
type PiCodingAgentHarnessWithOutput<TOutput extends JsonValue> = PiCodingAgentHarnessOptions & {
|
||||
output: (args: { response: string; session: AgentSession }) => TOutput | Promise<TOutput>;
|
||||
};
|
||||
|
||||
export function resolveModelSelection(
|
||||
explicitModel: PiCodingAgentModelSelection | undefined,
|
||||
environment: { PI_PROVIDER?: string; PI_MODEL?: string } = process.env,
|
||||
): PiCodingAgentModelSelection {
|
||||
const provider = (explicitModel?.provider ?? environment.PI_PROVIDER)?.trim();
|
||||
const id = (explicitModel?.id ?? environment.PI_MODEL)?.trim();
|
||||
if (!provider || !id) {
|
||||
throw new Error("Select a harness model explicitly or set both PI_PROVIDER and PI_MODEL as defaults.");
|
||||
}
|
||||
return { provider, id };
|
||||
}
|
||||
|
||||
function toTranscriptEvents(messages: AgentSession["messages"]): TranscriptEvent[] {
|
||||
const events: TranscriptEvent[] = [];
|
||||
for (const message of messages) {
|
||||
if (message.role === "user") {
|
||||
events.push({ type: "message", role: "user", content: contentText(message.content) });
|
||||
} else if (message.role === "assistant") {
|
||||
const text = contentText(message.content);
|
||||
if (text) events.push({ type: "message", role: "assistant", content: text });
|
||||
for (const part of message.content) {
|
||||
if (part.type === "toolCall") {
|
||||
events.push({
|
||||
type: "tool_call",
|
||||
id: part.id,
|
||||
name: part.name,
|
||||
arguments: normalizeRecord(part.arguments),
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (message.role === "toolResult") {
|
||||
const text = contentText(message.content);
|
||||
events.push({
|
||||
type: "tool_result",
|
||||
toolCallId: message.toolCallId,
|
||||
name: message.toolName,
|
||||
content: message.content.every((part) => part.type === "text") ? text : toJsonValue(message.content),
|
||||
...(message.isError ? { error: { message: text || "Tool failed" } } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
async function promptAgent(session: AgentSession, input: string, signal: AbortSignal | undefined): Promise<string> {
|
||||
signal?.throwIfAborted();
|
||||
const previousMessageCount = session.messages.length;
|
||||
await session.prompt(input);
|
||||
const assistant = session.messages
|
||||
.slice(previousMessageCount)
|
||||
.reverse()
|
||||
.find((message) => message.role === "assistant");
|
||||
if (!assistant) throw new Error("Agent run completed without an assistant message.");
|
||||
if (assistant.stopReason !== "stop") {
|
||||
throw new Error(
|
||||
assistant.errorMessage ?? `Agent run ended with unexpected stop reason: ${assistant.stopReason}.`,
|
||||
);
|
||||
}
|
||||
const output = session.getLastAssistantText();
|
||||
if (!output) throw new Error("Agent run produced no assistant text.");
|
||||
return output;
|
||||
}
|
||||
|
||||
async function runPiCodingAgent<TOutput extends JsonValue>(
|
||||
input: PiCodingAgentInput,
|
||||
signal: AbortSignal | undefined,
|
||||
setArtifact: HarnessContext["setArtifact"],
|
||||
options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput<TOutput>,
|
||||
): Promise<SimpleHarnessResult<string | TOutput>> {
|
||||
const startedAt = performance.now();
|
||||
signal?.throwIfAborted();
|
||||
const selection = resolveModelSelection(options.model);
|
||||
const modelRuntime = await ModelRuntime.create();
|
||||
const model = modelRuntime.getModel(selection.provider, selection.id);
|
||||
if (!model) throw new Error(`Eval model not found: ${selection.provider}/${selection.id}`);
|
||||
|
||||
const root = await mkdtemp(join(tmpdir(), "pi-eval-"));
|
||||
const cwd = join(root, "workspace");
|
||||
const agentDir = join(root, "agent");
|
||||
let transformedSystemPrompt: string | undefined;
|
||||
let sessionManager: SessionManager | undefined;
|
||||
let session: AgentSession | undefined;
|
||||
let outcome: { success: true; result: SimpleHarnessResult<string | TOutput> } | { success: false; error: unknown };
|
||||
try {
|
||||
await Promise.all([mkdir(cwd), mkdir(agentDir)]);
|
||||
const services = await createAgentSessionServices({
|
||||
cwd,
|
||||
agentDir,
|
||||
modelRuntime,
|
||||
settingsManager: SettingsManager.inMemory(),
|
||||
...(options.transformSystemPrompt
|
||||
? { resourceLoaderOptions: { systemPromptOverride: () => transformedSystemPrompt } }
|
||||
: {}),
|
||||
});
|
||||
signal?.throwIfAborted();
|
||||
sessionManager = SessionManager.create(cwd, join(root, "sessions"));
|
||||
setArtifact("runId", sessionManager.getSessionId());
|
||||
session = (
|
||||
await createAgentSessionFromServices({
|
||||
services,
|
||||
sessionManager,
|
||||
model,
|
||||
thinkingLevel: "off",
|
||||
noTools: options.noTools,
|
||||
})
|
||||
).session;
|
||||
|
||||
const evalSession = session;
|
||||
if (options.transformSystemPrompt) {
|
||||
transformedSystemPrompt = options.transformSystemPrompt(evalSession.systemPrompt);
|
||||
if (!transformedSystemPrompt.trim()) throw new Error("Transformed eval system prompt must not be empty.");
|
||||
await evalSession.reload();
|
||||
}
|
||||
let abortPromise: Promise<void> | undefined;
|
||||
const abort = () => {
|
||||
abortPromise ??= evalSession.abort();
|
||||
};
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
if (evalSession.extensionRunner.getExtensionPaths().length !== 0) {
|
||||
throw new Error("Expected an isolated eval session to start without extensions.");
|
||||
}
|
||||
const steps = typeof input === "string" ? [{ type: "prompt" as const, content: input }] : input;
|
||||
let response: string | undefined;
|
||||
for (const step of steps) {
|
||||
if (step.type === "prompt") {
|
||||
response = await promptAgent(evalSession, step.content, signal);
|
||||
} else {
|
||||
await evalSession.reload();
|
||||
}
|
||||
}
|
||||
if (response === undefined) throw new Error("Pi eval input must include at least one prompt step.");
|
||||
const output = "output" in options ? await options.output({ response, session: evalSession }) : response;
|
||||
const stats = evalSession.getSessionStats();
|
||||
const hasPricing = [model.cost, ...(model.cost.tiers ?? [])].some(
|
||||
({ input, output, cacheRead, cacheWrite }) => input > 0 || output > 0 || cacheRead > 0 || cacheWrite > 0,
|
||||
);
|
||||
outcome = {
|
||||
success: true,
|
||||
result: {
|
||||
output,
|
||||
events: toTranscriptEvents(evalSession.messages),
|
||||
usage: {
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
inputTokens: stats.tokens.input,
|
||||
outputTokens: stats.tokens.output,
|
||||
totalTokens: stats.tokens.total,
|
||||
toolCalls: stats.toolCalls,
|
||||
metadata: {
|
||||
cacheReadTokens: stats.tokens.cacheRead,
|
||||
cacheWriteTokens: stats.tokens.cacheWrite,
|
||||
...(hasPricing ? { estimatedCostUsd: stats.cost } : {}),
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
signal?.removeEventListener("abort", abort);
|
||||
if (abortPromise) await abortPromise;
|
||||
}
|
||||
} catch (error) {
|
||||
outcome = { success: false, error };
|
||||
}
|
||||
|
||||
const cleanupErrors: unknown[] = [];
|
||||
if (sessionManager) {
|
||||
try {
|
||||
const sessionPath = sessionManager.getSessionFile();
|
||||
if (sessionPath && existsSync(sessionPath)) {
|
||||
setArtifact(PI_SESSION_SNAPSHOT_ARTIFACT, await readFile(sessionPath, "utf8"));
|
||||
}
|
||||
} catch (error) {
|
||||
cleanupErrors.push(error);
|
||||
}
|
||||
}
|
||||
try {
|
||||
session?.dispose();
|
||||
} catch (error) {
|
||||
cleanupErrors.push(error);
|
||||
}
|
||||
try {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
} catch (error) {
|
||||
cleanupErrors.push(error);
|
||||
}
|
||||
|
||||
if (!outcome.success) {
|
||||
if (cleanupErrors.length === 0) throw outcome.error;
|
||||
throw new AggregateError([outcome.error, ...cleanupErrors], "Agent run failed and cleanup also failed.");
|
||||
}
|
||||
if (cleanupErrors.length === 1) throw cleanupErrors[0];
|
||||
if (cleanupErrors.length > 1) throw new AggregateError(cleanupErrors, "Agent cleanup failed.");
|
||||
return {
|
||||
...outcome.result,
|
||||
timings: { totalMs: performance.now() - startedAt },
|
||||
};
|
||||
}
|
||||
|
||||
export function createPiCodingAgentHarness<TOutput extends JsonValue>(
|
||||
options: PiCodingAgentHarnessWithOutput<TOutput>,
|
||||
): Harness<PiCodingAgentInput, TOutput>;
|
||||
export function createPiCodingAgentHarness(options?: PiCodingAgentHarnessOptions): Harness<PiCodingAgentInput, string>;
|
||||
export function createPiCodingAgentHarness<TOutput extends JsonValue>(
|
||||
options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput<TOutput> = {},
|
||||
) {
|
||||
return createHarness<PiCodingAgentInput, string | TOutput>({
|
||||
name: options.name ?? "pi-coding-agent",
|
||||
run: ({ input, signal, setArtifact }) => runPiCodingAgent(input, signal, setArtifact, options),
|
||||
});
|
||||
}
|
||||
59
third_party/pi/packages/evals/src/plan.ts
vendored
Normal file
59
third_party/pi/packages/evals/src/plan.ts
vendored
Normal file
@@ -0,0 +1,59 @@
|
||||
export const DOCUMENTATION_VARIANTS = ["without_docs", "with_docs"] as const;
|
||||
export type DocumentationVariant = (typeof DOCUMENTATION_VARIANTS)[number];
|
||||
|
||||
export type DiscoveredEvalCase = {
|
||||
file: string;
|
||||
fullName: string;
|
||||
evalSet: string;
|
||||
caseId: string;
|
||||
};
|
||||
|
||||
export type EvalTask = DiscoveredEvalCase & {
|
||||
variant: DocumentationVariant;
|
||||
model: string;
|
||||
runNumber: number;
|
||||
};
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export function parseDiscoveredCases(value: unknown): DiscoveredEvalCase[] {
|
||||
if (!Array.isArray(value)) throw new TypeError("Discovered eval cases must be an array.");
|
||||
const identities = new Set<string>();
|
||||
return value.map((item) => {
|
||||
if (!isRecord(item) || typeof item.name !== "string" || typeof item.file !== "string") {
|
||||
throw new TypeError("Discovered eval case is invalid.");
|
||||
}
|
||||
const [evalSet, caseId, ...extra] = item.name.split(" > ");
|
||||
if (!evalSet?.trim() || !caseId?.trim() || extra.length > 0) {
|
||||
throw new TypeError(`Documentation eval must use "<eval set> > <case>": ${item.name}`);
|
||||
}
|
||||
const identity = JSON.stringify([evalSet, caseId]);
|
||||
if (identities.has(identity)) throw new TypeError(`Duplicate eval case identity: ${item.name}`);
|
||||
identities.add(identity);
|
||||
return { file: item.file, fullName: item.name, evalSet, caseId };
|
||||
});
|
||||
}
|
||||
|
||||
export function createTaskPlan(
|
||||
cases: readonly DiscoveredEvalCase[],
|
||||
model: string,
|
||||
runsPerVariant: number,
|
||||
): EvalTask[] {
|
||||
if (!model.includes("/") || model.startsWith("/") || model.endsWith("/")) {
|
||||
throw new TypeError("Model identity must contain a provider and model.");
|
||||
}
|
||||
if (!Number.isSafeInteger(runsPerVariant) || runsPerVariant < 1) {
|
||||
throw new TypeError("Runs per variant must be a positive integer.");
|
||||
}
|
||||
const tasks: EvalTask[] = [];
|
||||
for (const evalCase of cases) {
|
||||
for (let runNumber = 1; runNumber <= runsPerVariant; runNumber += 1) {
|
||||
const variants: DocumentationVariant[] =
|
||||
runNumber % 2 === 1 ? ["without_docs", "with_docs"] : ["with_docs", "without_docs"];
|
||||
for (const variant of variants) tasks.push({ ...evalCase, variant, model, runNumber });
|
||||
}
|
||||
}
|
||||
return tasks;
|
||||
}
|
||||
483
third_party/pi/packages/evals/src/report.ts
vendored
Normal file
483
third_party/pi/packages/evals/src/report.ts
vendored
Normal file
@@ -0,0 +1,483 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { styleText } from "node:util";
|
||||
import type { ReportCase } from "@vitest-evals/core";
|
||||
import { readReportWorkspace, readVitestJsonReportFile } from "@vitest-evals/core/node";
|
||||
import type { DocumentationVariant, EvalTask } from "./plan.ts";
|
||||
|
||||
export const PI_SESSION_SNAPSHOT_ARTIFACT = "piSessionJsonl";
|
||||
|
||||
export type EvalRunIdentity = {
|
||||
evalSet: string;
|
||||
caseId: string;
|
||||
variant: DocumentationVariant;
|
||||
model: string;
|
||||
runNumber: number;
|
||||
};
|
||||
|
||||
export type ExpectedEvalRun = EvalRunIdentity;
|
||||
|
||||
type EvalMetrics = {
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
cacheReadTokens?: number;
|
||||
cacheWriteTokens?: number;
|
||||
totalTokens?: number;
|
||||
toolCalls?: number;
|
||||
totalMs?: number;
|
||||
estimatedCostUsd?: number;
|
||||
};
|
||||
|
||||
export type EvalObservation = EvalRunIdentity &
|
||||
EvalMetrics &
|
||||
({ outcome: "scored"; score: number } | { outcome: "unscored" | "skipped" | "pending" | "errored" });
|
||||
|
||||
export type PairedMetricSummary = {
|
||||
eligiblePairs: number;
|
||||
controlMean: number | null;
|
||||
treatmentMean: number | null;
|
||||
meanDelta: number | null;
|
||||
};
|
||||
|
||||
export type EvalSetComparison = {
|
||||
evalSet: string;
|
||||
totalPairs: number;
|
||||
eligiblePairs: number;
|
||||
blockedPairs: number;
|
||||
controlPassRate: number | null;
|
||||
treatmentPassRate: number | null;
|
||||
lift: number | null;
|
||||
flags: Array<"no-lift" | "negative-delta" | "control-saturated" | "treatment-saturated" | "flaky">;
|
||||
totalTokens: PairedMetricSummary;
|
||||
toolCalls: PairedMetricSummary;
|
||||
totalMs: PairedMetricSummary;
|
||||
estimatedCostUsd: PairedMetricSummary;
|
||||
};
|
||||
|
||||
export type BlockedPair = Omit<EvalRunIdentity, "variant"> & { reasons: string[] };
|
||||
|
||||
export type OperationalMetricTotal = { availableRuns: number; total: number | null };
|
||||
|
||||
export type VariantTotals = {
|
||||
variant: DocumentationVariant;
|
||||
runs: number;
|
||||
inputTokens: OperationalMetricTotal;
|
||||
outputTokens: OperationalMetricTotal;
|
||||
cacheReadTokens: OperationalMetricTotal;
|
||||
cacheWriteTokens: OperationalMetricTotal;
|
||||
totalTokens: OperationalMetricTotal;
|
||||
toolCalls: OperationalMetricTotal;
|
||||
totalMs: OperationalMetricTotal;
|
||||
estimatedCostUsd: OperationalMetricTotal;
|
||||
};
|
||||
|
||||
export type EvalComparisonReport = {
|
||||
schemaVersion: 3;
|
||||
protocolDigest: string;
|
||||
control: "without_docs";
|
||||
treatment: "with_docs";
|
||||
comparisons: EvalSetComparison[];
|
||||
blockedPairs: BlockedPair[];
|
||||
operationalTotals: VariantTotals[];
|
||||
};
|
||||
|
||||
function optionalMetric(value: unknown, name: string): number | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
||||
throw new TypeError(`${name} must be a finite non-negative number.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function validateScore(value: unknown): number | undefined {
|
||||
if (value === null || value === undefined) return undefined;
|
||||
if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
|
||||
throw new TypeError("Eval score must be between 0 and 1.");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function classifyCaseStatus(status: ReportCase["status"]): "errored" | "skipped" | "pending" | undefined {
|
||||
if (status === "failed") return "errored";
|
||||
if (status === "skipped" || status === "todo" || status === "disabled") return "skipped";
|
||||
if (status === "pending") return "pending";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function taskIdentity(task: EvalTask): EvalRunIdentity {
|
||||
return {
|
||||
evalSet: task.evalSet,
|
||||
caseId: task.caseId,
|
||||
variant: task.variant,
|
||||
model: task.model,
|
||||
runNumber: task.runNumber,
|
||||
};
|
||||
}
|
||||
|
||||
export function erroredObservation(task: EvalTask): EvalObservation {
|
||||
return { ...taskIdentity(task), outcome: "errored" };
|
||||
}
|
||||
|
||||
async function persistSession(caseResult: ReportCase, task: EvalTask, artifactDirectory: string): Promise<void> {
|
||||
const session = caseResult.harness?.run?.artifacts?.[PI_SESSION_SNAPSHOT_ARTIFACT];
|
||||
if (typeof session !== "string") return;
|
||||
const identity = JSON.stringify([task.evalSet, task.caseId, task.variant, task.model, task.runNumber]);
|
||||
const directory = join(
|
||||
artifactDirectory,
|
||||
task.variant,
|
||||
"sessions",
|
||||
createHash("sha256").update(identity).digest("hex"),
|
||||
);
|
||||
await mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
await writeFile(join(directory, "session.jsonl"), session, { mode: 0o600 });
|
||||
}
|
||||
|
||||
export async function readTaskObservation(
|
||||
task: EvalTask,
|
||||
reportPath: string,
|
||||
artifactDirectory: string,
|
||||
): Promise<EvalObservation> {
|
||||
const identity = taskIdentity(task);
|
||||
const loaded = await Promise.all([readReportWorkspace([reportPath]), readVitestJsonReportFile(reportPath)]).catch(
|
||||
() => undefined,
|
||||
);
|
||||
if (!loaded) return { ...identity, outcome: "errored" };
|
||||
const [{ workspace }, rawReport] = loaded;
|
||||
const reportedFullName = `${task.evalSet} ${task.caseId}`;
|
||||
const assertions = rawReport.testResults.flatMap(({ assertionResults }) => assertionResults);
|
||||
if (assertions.length !== 1) return { ...identity, outcome: "errored" };
|
||||
const assertion = assertions[0];
|
||||
if (assertion.fullName !== reportedFullName) return { ...identity, outcome: "errored" };
|
||||
const statusOutcome = classifyCaseStatus(assertion.status);
|
||||
if (statusOutcome === "skipped" || statusOutcome === "pending") return { ...identity, outcome: statusOutcome };
|
||||
if (workspace.cases.length !== 1) return { ...identity, outcome: "errored" };
|
||||
const caseResult = workspace.cases[0];
|
||||
if (caseResult.fullName !== reportedFullName) return { ...identity, outcome: "errored" };
|
||||
if (caseResult.status !== assertion.status) return { ...identity, outcome: "errored" };
|
||||
const run = caseResult.harness?.run;
|
||||
if (!run) return { ...identity, outcome: "errored" };
|
||||
await persistSession(caseResult, task, artifactDirectory);
|
||||
const actualModel = run.usage.provider && run.usage.model ? `${run.usage.provider}/${run.usage.model}` : undefined;
|
||||
if (actualModel !== task.model) return { ...identity, outcome: "errored" };
|
||||
let metrics: EvalMetrics = {};
|
||||
try {
|
||||
metrics = {
|
||||
inputTokens: optionalMetric(run.usage.inputTokens, "inputTokens"),
|
||||
outputTokens: optionalMetric(run.usage.outputTokens, "outputTokens"),
|
||||
cacheReadTokens: optionalMetric(run.usage.metadata?.cacheReadTokens, "cacheReadTokens"),
|
||||
cacheWriteTokens: optionalMetric(run.usage.metadata?.cacheWriteTokens, "cacheWriteTokens"),
|
||||
totalTokens: optionalMetric(run.usage.totalTokens, "totalTokens"),
|
||||
toolCalls: optionalMetric(run.usage.toolCalls, "toolCalls"),
|
||||
totalMs: optionalMetric(run.timings?.totalMs, "totalMs"),
|
||||
estimatedCostUsd: optionalMetric(run.usage.metadata?.estimatedCostUsd, "estimatedCostUsd"),
|
||||
};
|
||||
} catch {
|
||||
return { ...identity, outcome: "errored" };
|
||||
}
|
||||
if (statusOutcome === "errored" || run.errors.length > 0) {
|
||||
return { ...identity, ...metrics, outcome: "errored" };
|
||||
}
|
||||
let score: number | undefined;
|
||||
try {
|
||||
score = validateScore(caseResult.eval?.avgScore);
|
||||
} catch {
|
||||
return { ...identity, ...metrics, outcome: "errored" };
|
||||
}
|
||||
if (score === undefined) return { ...identity, ...metrics, outcome: "unscored" };
|
||||
return { ...identity, ...metrics, outcome: "scored", score };
|
||||
}
|
||||
|
||||
const CONTROL = "without_docs";
|
||||
const TREATMENT = "with_docs";
|
||||
const VARIANTS = [CONTROL, TREATMENT] as const;
|
||||
|
||||
type Pair = { control: EvalObservation & { outcome: "scored" }; treatment: EvalObservation & { outcome: "scored" } };
|
||||
type PairGroup = Omit<EvalRunIdentity, "variant"> & {
|
||||
expected: Map<DocumentationVariant, number>;
|
||||
observations: Map<DocumentationVariant, EvalObservation[]>;
|
||||
};
|
||||
|
||||
function mean(values: readonly number[]): number | null {
|
||||
return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
function difference(treatment: number, control: number): number {
|
||||
return Number((treatment - control).toPrecision(15));
|
||||
}
|
||||
|
||||
function pairKey(identity: Omit<EvalRunIdentity, "variant">): string {
|
||||
return JSON.stringify([identity.evalSet, identity.caseId, identity.model, identity.runNumber]);
|
||||
}
|
||||
|
||||
function groupPairs(expectedRuns: readonly ExpectedEvalRun[], observations: readonly EvalObservation[]): PairGroup[] {
|
||||
const groups = new Map<string, PairGroup>();
|
||||
const getGroup = (identity: EvalRunIdentity): PairGroup => {
|
||||
const key = pairKey(identity);
|
||||
const existing = groups.get(key);
|
||||
if (existing) return existing;
|
||||
const group = {
|
||||
evalSet: identity.evalSet,
|
||||
caseId: identity.caseId,
|
||||
model: identity.model,
|
||||
runNumber: identity.runNumber,
|
||||
expected: new Map<DocumentationVariant, number>(),
|
||||
observations: new Map<DocumentationVariant, EvalObservation[]>(),
|
||||
};
|
||||
groups.set(key, group);
|
||||
return group;
|
||||
};
|
||||
for (const expected of expectedRuns) {
|
||||
const group = getGroup(expected);
|
||||
group.expected.set(expected.variant, (group.expected.get(expected.variant) ?? 0) + 1);
|
||||
}
|
||||
for (const observation of observations) {
|
||||
const group = getGroup(observation);
|
||||
const runs = group.observations.get(observation.variant) ?? [];
|
||||
runs.push(observation);
|
||||
group.observations.set(observation.variant, runs);
|
||||
}
|
||||
return [...groups.values()].sort(
|
||||
(left, right) =>
|
||||
left.evalSet.localeCompare(right.evalSet) ||
|
||||
left.caseId.localeCompare(right.caseId) ||
|
||||
left.model.localeCompare(right.model) ||
|
||||
left.runNumber - right.runNumber,
|
||||
);
|
||||
}
|
||||
|
||||
function resolvePair(group: PairGroup): { pair?: Pair; blocked?: BlockedPair } {
|
||||
const reasons: string[] = [];
|
||||
for (const variant of VARIANTS) {
|
||||
const expected = group.expected.get(variant) ?? 0;
|
||||
const observed = group.observations.get(variant) ?? [];
|
||||
if (expected !== 1) reasons.push(`${variant}: design expected 1 run, found ${expected}`);
|
||||
if (observed.length !== expected) {
|
||||
reasons.push(
|
||||
`${variant}: expected ${expected} observation${expected === 1 ? "" : "s"}, found ${observed.length}`,
|
||||
);
|
||||
}
|
||||
if (expected === 1 && observed.length === 1 && observed[0].outcome !== "scored") {
|
||||
reasons.push(`${variant}: ${observed[0].outcome}`);
|
||||
}
|
||||
}
|
||||
if (reasons.length > 0) {
|
||||
return {
|
||||
blocked: {
|
||||
evalSet: group.evalSet,
|
||||
caseId: group.caseId,
|
||||
model: group.model,
|
||||
runNumber: group.runNumber,
|
||||
reasons,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
pair: {
|
||||
control: group.observations.get(CONTROL)![0] as EvalObservation & { outcome: "scored" },
|
||||
treatment: group.observations.get(TREATMENT)![0] as EvalObservation & { outcome: "scored" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeMetric(
|
||||
pairs: readonly Pair[],
|
||||
select: (observation: EvalObservation) => number | undefined,
|
||||
): PairedMetricSummary {
|
||||
const control: number[] = [];
|
||||
const treatment: number[] = [];
|
||||
for (const pair of pairs) {
|
||||
const controlValue = select(pair.control);
|
||||
const treatmentValue = select(pair.treatment);
|
||||
if (controlValue === undefined || treatmentValue === undefined) continue;
|
||||
control.push(controlValue);
|
||||
treatment.push(treatmentValue);
|
||||
}
|
||||
const controlMean = mean(control);
|
||||
const treatmentMean = mean(treatment);
|
||||
return {
|
||||
eligiblePairs: control.length,
|
||||
controlMean,
|
||||
treatmentMean,
|
||||
meanDelta: controlMean === null || treatmentMean === null ? null : difference(treatmentMean, controlMean),
|
||||
};
|
||||
}
|
||||
|
||||
function operationalTotal(
|
||||
runs: readonly EvalObservation[],
|
||||
select: (observation: EvalObservation) => number | undefined,
|
||||
): OperationalMetricTotal {
|
||||
const values = runs.flatMap((run) => {
|
||||
const value = select(run);
|
||||
return value === undefined ? [] : [value];
|
||||
});
|
||||
return {
|
||||
availableRuns: values.length,
|
||||
total: values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0),
|
||||
};
|
||||
}
|
||||
|
||||
function variantTotals(observations: readonly EvalObservation[], variant: DocumentationVariant): VariantTotals {
|
||||
const runs = observations.filter((observation) => observation.variant === variant);
|
||||
return {
|
||||
variant,
|
||||
runs: runs.length,
|
||||
inputTokens: operationalTotal(runs, ({ inputTokens }) => inputTokens),
|
||||
outputTokens: operationalTotal(runs, ({ outputTokens }) => outputTokens),
|
||||
cacheReadTokens: operationalTotal(runs, ({ cacheReadTokens }) => cacheReadTokens),
|
||||
cacheWriteTokens: operationalTotal(runs, ({ cacheWriteTokens }) => cacheWriteTokens),
|
||||
totalTokens: operationalTotal(runs, ({ totalTokens }) => totalTokens),
|
||||
toolCalls: operationalTotal(runs, ({ toolCalls }) => toolCalls),
|
||||
totalMs: operationalTotal(runs, ({ totalMs }) => totalMs),
|
||||
estimatedCostUsd: operationalTotal(runs, ({ estimatedCostUsd }) => estimatedCostUsd),
|
||||
};
|
||||
}
|
||||
|
||||
function comparisonFlags(pairs: readonly Pair[], controlPassRate: number | null, treatmentPassRate: number | null) {
|
||||
const flags: Array<"no-lift" | "negative-delta" | "control-saturated" | "treatment-saturated" | "flaky"> = [];
|
||||
if (controlPassRate !== null && treatmentPassRate !== null) {
|
||||
if (controlPassRate === treatmentPassRate) flags.push("no-lift");
|
||||
if (treatmentPassRate < controlPassRate) flags.push("negative-delta");
|
||||
if (controlPassRate === 1) flags.push("control-saturated");
|
||||
if (treatmentPassRate === 1) flags.push("treatment-saturated");
|
||||
}
|
||||
const outcomes = new Map<string, Set<string>>();
|
||||
for (const pair of pairs) {
|
||||
const key = JSON.stringify([pair.control.caseId, pair.control.variant]);
|
||||
const control = outcomes.get(key) ?? new Set<string>();
|
||||
control.add(String(pair.control.score >= 1));
|
||||
outcomes.set(key, control);
|
||||
const treatmentKey = JSON.stringify([pair.treatment.caseId, pair.treatment.variant]);
|
||||
const treatment = outcomes.get(treatmentKey) ?? new Set<string>();
|
||||
treatment.add(String(pair.treatment.score >= 1));
|
||||
outcomes.set(treatmentKey, treatment);
|
||||
}
|
||||
if ([...outcomes.values()].some((values) => values.size > 1)) flags.push("flaky");
|
||||
return flags;
|
||||
}
|
||||
|
||||
export function summarizeEvalObservations(
|
||||
protocolDigest: string,
|
||||
expectedRuns: readonly ExpectedEvalRun[],
|
||||
observations: readonly EvalObservation[],
|
||||
): EvalComparisonReport {
|
||||
const groups = groupPairs(expectedRuns, observations);
|
||||
const blockedPairs: BlockedPair[] = [];
|
||||
const pairsByEvalSet = new Map<string, Pair[]>();
|
||||
const totalsByEvalSet = new Map<string, number>();
|
||||
for (const group of groups) {
|
||||
totalsByEvalSet.set(group.evalSet, (totalsByEvalSet.get(group.evalSet) ?? 0) + 1);
|
||||
const { pair, blocked } = resolvePair(group);
|
||||
if (blocked) blockedPairs.push(blocked);
|
||||
if (pair) pairsByEvalSet.set(group.evalSet, [...(pairsByEvalSet.get(group.evalSet) ?? []), pair]);
|
||||
}
|
||||
const comparisons = [...totalsByEvalSet]
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([evalSet, totalPairs]) => {
|
||||
const pairs = pairsByEvalSet.get(evalSet) ?? [];
|
||||
const blockedPairCount = totalPairs - pairs.length;
|
||||
const publishHeadline = blockedPairCount === 0 && pairs.length > 0;
|
||||
const controlPassRate = publishHeadline
|
||||
? pairs.filter(({ control }) => control.score >= 1).length / pairs.length
|
||||
: null;
|
||||
const treatmentPassRate = publishHeadline
|
||||
? pairs.filter(({ treatment }) => treatment.score >= 1).length / pairs.length
|
||||
: null;
|
||||
return {
|
||||
evalSet,
|
||||
totalPairs,
|
||||
eligiblePairs: pairs.length,
|
||||
blockedPairs: blockedPairCount,
|
||||
controlPassRate,
|
||||
treatmentPassRate,
|
||||
lift:
|
||||
controlPassRate === null || treatmentPassRate === null
|
||||
? null
|
||||
: difference(treatmentPassRate, controlPassRate),
|
||||
flags: comparisonFlags(pairs, controlPassRate, treatmentPassRate),
|
||||
totalTokens: summarizeMetric(pairs, ({ totalTokens }) => totalTokens),
|
||||
toolCalls: summarizeMetric(pairs, ({ toolCalls }) => toolCalls),
|
||||
totalMs: summarizeMetric(pairs, ({ totalMs }) => totalMs),
|
||||
estimatedCostUsd: summarizeMetric(pairs, ({ estimatedCostUsd }) => estimatedCostUsd),
|
||||
};
|
||||
});
|
||||
return {
|
||||
schemaVersion: 3,
|
||||
protocolDigest,
|
||||
control: CONTROL,
|
||||
treatment: TREATMENT,
|
||||
comparisons,
|
||||
blockedPairs,
|
||||
operationalTotals: [variantTotals(observations, CONTROL), variantTotals(observations, TREATMENT)],
|
||||
};
|
||||
}
|
||||
|
||||
function percentage(value: number | null): string {
|
||||
return value === null ? "unavailable" : `${(value * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function signed(value: number, digits: number): string {
|
||||
return `${value >= 0 ? "+" : ""}${value.toFixed(digits)}`;
|
||||
}
|
||||
|
||||
function pairedMetric(label: string, metric: PairedMetricSummary, unit = ""): string {
|
||||
if (metric.meanDelta === null || metric.controlMean === null || metric.treatmentMean === null) {
|
||||
return ` ${label.padStart(10)} unavailable`;
|
||||
}
|
||||
return ` ${label.padStart(10)} ${signed(metric.meanDelta, 1)}${unit} (with ${metric.treatmentMean.toFixed(1)}${unit}, without ${metric.controlMean.toFixed(1)}${unit}, ${metric.eligiblePairs} pairs)`;
|
||||
}
|
||||
|
||||
function operationalMetric(metric: OperationalMetricTotal, runs: number, format: (total: number) => string): string {
|
||||
if (metric.total === null) return `unavailable (0/${runs} measured)`;
|
||||
const coverage = metric.availableRuns === runs ? "" : ` (${metric.availableRuns}/${runs} measured)`;
|
||||
return `${format(metric.total)}${coverage}`;
|
||||
}
|
||||
|
||||
export function formatEvalComparisonReport(report: EvalComparisonReport): string {
|
||||
if (report.comparisons.length === 0) return "";
|
||||
const lines = [styleText("bold", "Documentation Eval Comparisons")];
|
||||
for (const comparison of report.comparisons) {
|
||||
lines.push(` ${comparison.evalSet}`);
|
||||
lines.push(` Pairs ${comparison.eligiblePairs}/${comparison.totalPairs} eligible`);
|
||||
if (comparison.lift === null) {
|
||||
lines.push(
|
||||
comparison.blockedPairs > 0
|
||||
? " Pass rate withheld because pairs are blocked"
|
||||
: " Pass rate unavailable",
|
||||
);
|
||||
} else {
|
||||
lines.push(
|
||||
` Pass rate ${signed(comparison.lift * 100, 1)} pp (with ${percentage(comparison.treatmentPassRate)}, without ${percentage(comparison.controlPassRate)})`,
|
||||
);
|
||||
}
|
||||
if (comparison.flags.length > 0) lines.push(` Flags ${comparison.flags.join(", ")}`);
|
||||
lines.push(pairedMetric("Tokens", comparison.totalTokens));
|
||||
lines.push(pairedMetric("Tools", comparison.toolCalls));
|
||||
lines.push(pairedMetric("Latency", comparison.totalMs, "ms"));
|
||||
const cost = comparison.estimatedCostUsd;
|
||||
if (cost.meanDelta === null || cost.controlMean === null || cost.treatmentMean === null) {
|
||||
lines.push(" Est. cost unavailable");
|
||||
} else {
|
||||
lines.push(
|
||||
` Est. cost ${cost.meanDelta >= 0 ? "+" : "-"}$${Math.abs(cost.meanDelta).toFixed(4)} (with $${cost.treatmentMean.toFixed(4)}, without $${cost.controlMean.toFixed(4)}, ${cost.eligiblePairs} pairs)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push(" Operational totals");
|
||||
for (const totals of report.operationalTotals) {
|
||||
const tokens = operationalMetric(totals.totalTokens, totals.runs, (total) => `${total} tokens`);
|
||||
const tools = operationalMetric(totals.toolCalls, totals.runs, (total) => `${total} tools`);
|
||||
const latency = operationalMetric(totals.totalMs, totals.runs, (total) => `${(total / 1000).toFixed(2)}s`);
|
||||
const cost = operationalMetric(totals.estimatedCostUsd, totals.runs, (total) => `$${total.toFixed(4)} cost`);
|
||||
lines.push(` ${totals.variant}: ${totals.runs} runs, ${tokens}, ${tools}, ${latency}, ${cost}`);
|
||||
}
|
||||
if (report.blockedPairs.length > 0) {
|
||||
lines.push(" Blocked pairs");
|
||||
for (const blocked of report.blockedPairs) {
|
||||
lines.push(
|
||||
` ${blocked.evalSet}/${blocked.caseId}/${blocked.model}/run-${blocked.runNumber}: ${blocked.reasons.join("; ")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
17
third_party/pi/packages/evals/src/smoke.eval.ts
vendored
17
third_party/pi/packages/evals/src/smoke.eval.ts
vendored
@@ -1,17 +0,0 @@
|
||||
import { expect } from "vitest";
|
||||
import { describeEval } from "vitest-evals";
|
||||
import { createPiCodingAgentHarness } from "./pi-harness.ts";
|
||||
|
||||
const piCodingAgentHarness = createPiCodingAgentHarness({ noTools: "all" });
|
||||
|
||||
describeEval("Pi Coding Agent smoke", { harness: piCodingAgentHarness }, (it) => {
|
||||
it("runs a basic prompt end to end", async ({ run }) => {
|
||||
const result = await run("What's the capital of France? Respond with only the city name.");
|
||||
|
||||
expect(result.output.trim()).toBe("Paris");
|
||||
expect(result.errors).toEqual([]);
|
||||
expect(result.usage.provider).toBe(process.env.PI_PROVIDER);
|
||||
expect(result.usage.model).toBe(process.env.PI_MODEL);
|
||||
expect(result.usage.totalTokens).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -1,113 +0,0 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { basename, join, relative } from "node:path";
|
||||
import {
|
||||
type RunnerTestCase,
|
||||
recordArtifact,
|
||||
type TestArtifact,
|
||||
type TestArtifactBase,
|
||||
type TestAttachment,
|
||||
} from "vitest";
|
||||
import type { HarnessRun } from "vitest-evals/harness";
|
||||
|
||||
export const PI_SESSION_SNAPSHOT_ARTIFACT = "piSessionJsonl";
|
||||
|
||||
const evalSessionArtifactKey = Symbol("pi-evals-session-artifact");
|
||||
const evalSourceArtifactKey = Symbol("pi-evals-source-artifact");
|
||||
|
||||
interface PiSessionAttachment extends TestAttachment {
|
||||
name: "session.jsonl";
|
||||
contentType: "application/jsonl";
|
||||
body: string;
|
||||
bodyEncoding: "utf-8";
|
||||
}
|
||||
|
||||
export interface SourceAttachment extends TestAttachment {
|
||||
name: string;
|
||||
contentType: string;
|
||||
body: string;
|
||||
bodyEncoding: "utf-8";
|
||||
}
|
||||
|
||||
interface PiSessionArtifact extends TestArtifactBase {
|
||||
type: "@earendil-works/pi-evals:session";
|
||||
runId: string;
|
||||
attachments: [PiSessionAttachment] | [];
|
||||
}
|
||||
|
||||
interface SourceArtifact extends TestArtifactBase {
|
||||
type: "@earendil-works/pi-evals:source";
|
||||
runId: string;
|
||||
attachments: [SourceAttachment] | [];
|
||||
}
|
||||
|
||||
declare module "vitest" {
|
||||
interface TestArtifactRegistry {
|
||||
[evalSessionArtifactKey]: PiSessionArtifact;
|
||||
[evalSourceArtifactKey]: SourceArtifact;
|
||||
}
|
||||
}
|
||||
|
||||
export async function recordEvalSessionArtifact(
|
||||
task: Readonly<RunnerTestCase>,
|
||||
run: Pick<HarnessRun, "artifacts">,
|
||||
): Promise<void> {
|
||||
const runId = run.artifacts?.runId;
|
||||
const session = run.artifacts?.[PI_SESSION_SNAPSHOT_ARTIFACT];
|
||||
if (session === undefined) return;
|
||||
if (typeof runId !== "string" || typeof session !== "string") {
|
||||
throw new TypeError("Pi eval session artifact metadata is invalid.");
|
||||
}
|
||||
await recordArtifact(task, {
|
||||
type: "@earendil-works/pi-evals:session",
|
||||
runId,
|
||||
attachments: [
|
||||
{
|
||||
name: "session.jsonl",
|
||||
contentType: "application/jsonl",
|
||||
body: session,
|
||||
bodyEncoding: "utf-8",
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
export async function recordEvalSourceArtifact(
|
||||
task: Readonly<RunnerTestCase>,
|
||||
runId: string,
|
||||
attachment: SourceAttachment,
|
||||
): Promise<void> {
|
||||
await recordArtifact(task, {
|
||||
type: "@earendil-works/pi-evals:source",
|
||||
runId,
|
||||
attachments: [attachment],
|
||||
});
|
||||
}
|
||||
|
||||
export async function persistEvalArtifactReferences(
|
||||
artifacts: ReadonlyArray<TestArtifact>,
|
||||
runId: string,
|
||||
artifactDirectory: string,
|
||||
): Promise<Array<{ name: string; path: string }>> {
|
||||
const references: Array<{ name: string; path: string }> = [];
|
||||
for (const artifact of artifacts) {
|
||||
if (
|
||||
(artifact.type !== "@earendil-works/pi-evals:session" &&
|
||||
artifact.type !== "@earendil-works/pi-evals:source") ||
|
||||
artifact.runId !== runId
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const category = artifact.type === "@earendil-works/pi-evals:session" ? "sessions" : "sources";
|
||||
for (const attachment of artifact.attachments) {
|
||||
const name = basename(attachment.name);
|
||||
if (name !== attachment.name) throw new TypeError(`Invalid eval artifact name: ${attachment.name}`);
|
||||
const directory = join(artifactDirectory, category, createHash("sha256").update(runId).digest("hex"));
|
||||
await mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const path = join(directory, name);
|
||||
await writeFile(path, attachment.body, { encoding: "utf8", mode: 0o600 });
|
||||
references.push({ name, path: relative(artifactDirectory, path) });
|
||||
}
|
||||
}
|
||||
return references;
|
||||
}
|
||||
@@ -1,193 +0,0 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
attachHarnessRunToError,
|
||||
getHarnessRunFromError,
|
||||
type Harness,
|
||||
type HarnessRun,
|
||||
type JsonValue,
|
||||
} from "vitest-evals/harness";
|
||||
|
||||
export const EVAL_HARNESS_ITERATION_ARTIFACT = "vitestEvalsHarnessIteration";
|
||||
|
||||
export type EvalHarnessIterationArtifact = {
|
||||
schemaVersion: 1;
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
harness: string;
|
||||
baseline: string;
|
||||
candidates: string[];
|
||||
repetition: number;
|
||||
};
|
||||
|
||||
export type EvalHarnessTableRow<TInput, TOutput extends JsonValue | undefined> = {
|
||||
harness: Harness<TInput, TOutput>;
|
||||
name: string;
|
||||
repetition: number;
|
||||
};
|
||||
|
||||
export type EvalHarnessTablePairOptions<TInput, TOutput extends JsonValue | undefined> = {
|
||||
baseline: Harness<TInput, TOutput>;
|
||||
candidate: Harness<TInput, TOutput>;
|
||||
repetitions?: number;
|
||||
};
|
||||
|
||||
export type EvalHarnessTableCandidatesOptions<TInput, TOutput extends JsonValue | undefined> = {
|
||||
baseline: Harness<TInput, TOutput>;
|
||||
candidates: readonly Harness<TInput, TOutput>[];
|
||||
repetitions?: number;
|
||||
};
|
||||
|
||||
export type EvalHarnessTableOptions<TInput, TOutput extends JsonValue | undefined> =
|
||||
| EvalHarnessTablePairOptions<TInput, TOutput>
|
||||
| EvalHarnessTableCandidatesOptions<TInput, TOutput>;
|
||||
|
||||
type EvalHarnessIterationPlan = Omit<EvalHarnessIterationArtifact, "groupKey">;
|
||||
|
||||
export function parseEvalHarnessIterationArtifact(
|
||||
value: JsonValue | undefined,
|
||||
): EvalHarnessIterationArtifact | undefined {
|
||||
if (value === null || value === undefined || typeof value !== "object" || Array.isArray(value)) return undefined;
|
||||
const { schemaVersion, evalSet, groupKey, harness, baseline, candidates, repetition } = value;
|
||||
if (
|
||||
schemaVersion !== 1 ||
|
||||
typeof evalSet !== "string" ||
|
||||
typeof groupKey !== "string" ||
|
||||
typeof harness !== "string" ||
|
||||
typeof baseline !== "string" ||
|
||||
!Array.isArray(candidates) ||
|
||||
!candidates.every((name): name is string => typeof name === "string") ||
|
||||
typeof repetition !== "number"
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return { schemaVersion, evalSet, groupKey, harness, baseline, candidates, repetition };
|
||||
}
|
||||
|
||||
function canonicalizeJson(value: unknown, ancestors: WeakSet<object>): JsonValue {
|
||||
if (value === null || typeof value === "string" || typeof value === "boolean") return value;
|
||||
if (typeof value === "number") {
|
||||
if (!Number.isFinite(value)) throw new TypeError("Eval input must contain only finite numbers.");
|
||||
return value;
|
||||
}
|
||||
if (typeof value !== "object") throw new TypeError("Eval input must be JSON-serializable.");
|
||||
if (ancestors.has(value)) throw new TypeError("Eval input must not contain circular references.");
|
||||
|
||||
ancestors.add(value);
|
||||
try {
|
||||
if (Array.isArray(value)) {
|
||||
const result: JsonValue[] = [];
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
if (!Object.hasOwn(value, index)) throw new TypeError("Eval input arrays must not be sparse.");
|
||||
result.push(canonicalizeJson(value[index], ancestors));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
const prototype = Object.getPrototypeOf(value);
|
||||
if (prototype !== Object.prototype && prototype !== null) {
|
||||
throw new TypeError("Eval input must contain only plain objects and arrays.");
|
||||
}
|
||||
const entries: Array<[string, unknown]> = Object.entries(value);
|
||||
return Object.fromEntries(
|
||||
entries
|
||||
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
|
||||
.map(([key, item]): [string, JsonValue] => [key, canonicalizeJson(item, ancestors)]),
|
||||
);
|
||||
} finally {
|
||||
ancestors.delete(value);
|
||||
}
|
||||
}
|
||||
|
||||
function deriveInputKey(input: unknown): string {
|
||||
if (typeof input === "object" && input !== null && !Array.isArray(input) && "id" in input) {
|
||||
const id = input.id;
|
||||
if (typeof id === "string" && id.trim()) return id.trim();
|
||||
}
|
||||
const canonicalInput = JSON.stringify(canonicalizeJson(input, new WeakSet()));
|
||||
if (canonicalInput === undefined) throw new TypeError("Eval input must be JSON-serializable.");
|
||||
return createHash("sha256").update(canonicalInput).digest("hex");
|
||||
}
|
||||
|
||||
export function deriveEvalGroupKey(input: unknown, repetition: number): string {
|
||||
return JSON.stringify([deriveInputKey(input), repetition]);
|
||||
}
|
||||
|
||||
function validateOptions<TInput, TOutput extends JsonValue | undefined>(
|
||||
evalSet: string,
|
||||
baseline: Harness<TInput, TOutput>,
|
||||
candidates: readonly Harness<TInput, TOutput>[],
|
||||
repetitions: number,
|
||||
): void {
|
||||
if (!evalSet.trim()) throw new TypeError("evalSet must not be empty.");
|
||||
if (candidates.length === 0) throw new TypeError("At least one candidate harness is required.");
|
||||
const harnesses = [baseline, ...candidates];
|
||||
const names = new Set(harnesses.map((harness) => harness.name));
|
||||
if (names.size !== harnesses.length) throw new TypeError("Harness names must be unique within an eval set.");
|
||||
if (!Number.isSafeInteger(repetitions) || repetitions < 1) {
|
||||
throw new TypeError("repetitions must be a positive integer.");
|
||||
}
|
||||
}
|
||||
|
||||
function withIterationArtifact<TInput, TOutput extends JsonValue | undefined>(
|
||||
harness: Harness<TInput, TOutput>,
|
||||
plan: EvalHarnessIterationPlan,
|
||||
): Harness<TInput, TOutput> {
|
||||
return {
|
||||
name: harness.name,
|
||||
async run(input, context) {
|
||||
const groupKey = deriveEvalGroupKey(input, plan.repetition);
|
||||
const artifact: EvalHarnessIterationArtifact = { ...plan, groupKey };
|
||||
context.setArtifact(EVAL_HARNESS_ITERATION_ARTIFACT, artifact);
|
||||
const attachIterationArtifact = <TRun extends HarnessRun>(run: TRun): TRun => {
|
||||
run.artifacts = { ...context.artifacts, ...run.artifacts, [EVAL_HARNESS_ITERATION_ARTIFACT]: artifact };
|
||||
return run;
|
||||
};
|
||||
try {
|
||||
return attachIterationArtifact(await harness.run(input, context));
|
||||
} catch (error) {
|
||||
const partialRun = getHarnessRunFromError(error);
|
||||
if (partialRun) {
|
||||
throw attachHarnessRunToError(error, attachIterationArtifact(partialRun));
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
|
||||
evalSet: string,
|
||||
options: EvalHarnessTablePairOptions<TInput, TOutput>,
|
||||
): EvalHarnessTableRow<TInput, TOutput>[];
|
||||
export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
|
||||
evalSet: string,
|
||||
options: EvalHarnessTableCandidatesOptions<TInput, TOutput>,
|
||||
): EvalHarnessTableRow<TInput, TOutput>[];
|
||||
export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
|
||||
evalSet: string,
|
||||
options: EvalHarnessTableOptions<TInput, TOutput>,
|
||||
): EvalHarnessTableRow<TInput, TOutput>[] {
|
||||
const repetitions = options.repetitions ?? 1;
|
||||
const candidates = "candidate" in options ? [options.candidate] : options.candidates;
|
||||
validateOptions(evalSet, options.baseline, candidates, repetitions);
|
||||
|
||||
const rows: EvalHarnessTableRow<TInput, TOutput>[] = [];
|
||||
const harnesses = [options.baseline, ...candidates];
|
||||
for (let repetition = 1; repetition <= repetitions; repetition += 1) {
|
||||
for (const harness of harnesses) {
|
||||
const plan: EvalHarnessIterationPlan = {
|
||||
schemaVersion: 1,
|
||||
evalSet,
|
||||
harness: harness.name,
|
||||
baseline: options.baseline.name,
|
||||
candidates: candidates.map(({ name }) => name),
|
||||
repetition,
|
||||
};
|
||||
rows.push({
|
||||
harness: withIterationArtifact(harness, plan),
|
||||
name: harness.name,
|
||||
repetition,
|
||||
});
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { appendFile, mkdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { Reporter, SerializedError, TestCase, TestModule, TestRunEndReason, Vitest } from "vitest/node";
|
||||
import { isHarnessRun } from "vitest-evals/harness";
|
||||
import { PI_SESSION_SNAPSHOT_ARTIFACT, persistEvalArtifactReferences } from "./artifacts.ts";
|
||||
import { EVAL_HARNESS_ITERATION_ARTIFACT, parseEvalHarnessIterationArtifact } from "./harness-table.ts";
|
||||
import { formatHarnessComparisonReport, type HarnessObservation, summarizeHarnessComparisons } from "./summary.ts";
|
||||
|
||||
function readFiniteNumber(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
async function appendHarnessRunReport(test: TestCase): Promise<void> {
|
||||
const artifactDirectory = process.env.PI_EVAL_ARTIFACT_DIR?.trim();
|
||||
if (!artifactDirectory) return;
|
||||
const harness = test.meta().harness;
|
||||
if (!harness || !isHarnessRun(harness.run)) return;
|
||||
|
||||
const run = harness.run;
|
||||
const artifactRunId = run.artifacts?.runId;
|
||||
const runId = typeof artifactRunId === "string" ? artifactRunId : randomUUID();
|
||||
const metadata = Object.fromEntries(
|
||||
Object.entries(run.artifacts ?? {}).filter(([name]) => name !== "runId" && name !== PI_SESSION_SNAPSHOT_ARTIFACT),
|
||||
);
|
||||
const record = {
|
||||
schemaVersion: 1,
|
||||
runId,
|
||||
test: {
|
||||
id: test.id,
|
||||
file: test.module.relativeModuleId,
|
||||
name: test.name,
|
||||
fullName: test.fullName,
|
||||
status: test.result().state,
|
||||
},
|
||||
harness: harness.name,
|
||||
usage: run.usage,
|
||||
...(run.timings ? { timings: run.timings } : {}),
|
||||
...(run.errors.length > 0 ? { errors: run.errors } : {}),
|
||||
artifacts: await persistEvalArtifactReferences(test.artifacts(), runId, artifactDirectory),
|
||||
...(Object.keys(metadata).length > 0 ? { metadata } : {}),
|
||||
};
|
||||
await mkdir(artifactDirectory, { recursive: true, mode: 0o700 });
|
||||
await appendFile(join(artifactDirectory, "runs.jsonl"), `${JSON.stringify(record)}\n`, {
|
||||
encoding: "utf8",
|
||||
flag: "a",
|
||||
mode: 0o600,
|
||||
});
|
||||
}
|
||||
|
||||
function collectHarnessObservations(modules: ReadonlyArray<TestModule>): HarnessObservation[] {
|
||||
const observations: HarnessObservation[] = [];
|
||||
for (const module of modules) {
|
||||
for (const test of module.children.allTests()) {
|
||||
const harness = test.meta().harness;
|
||||
if (!harness || !isHarnessRun(harness.run)) continue;
|
||||
const run = harness.run;
|
||||
const iteration = parseEvalHarnessIterationArtifact(run.artifacts?.[EVAL_HARNESS_ITERATION_ARTIFACT]);
|
||||
if (!iteration) continue;
|
||||
const score = readFiniteNumber(test.meta().eval?.avgScore);
|
||||
const estimatedCostUsd = readFiniteNumber(run.usage.metadata?.estimatedCostUsd);
|
||||
const observation = {
|
||||
evalSet: iteration.evalSet,
|
||||
groupKey: iteration.groupKey,
|
||||
testName: test.name,
|
||||
file: module.relativeModuleId,
|
||||
harness: iteration.harness,
|
||||
baseline: iteration.baseline,
|
||||
candidates: iteration.candidates,
|
||||
repetition: iteration.repetition,
|
||||
...(run.usage.totalTokens === undefined ? {} : { totalTokens: run.usage.totalTokens }),
|
||||
...(run.timings?.totalMs === undefined ? {} : { totalMs: run.timings.totalMs }),
|
||||
...(estimatedCostUsd === undefined ? {} : { estimatedCostUsd }),
|
||||
};
|
||||
if (run.errors.length > 0) observations.push({ ...observation, outcome: "errored" });
|
||||
else if (score !== undefined) observations.push({ ...observation, outcome: "scored", score });
|
||||
else {
|
||||
const state = test.result().state;
|
||||
const outcome = state === "passed" ? "unscored" : state === "failed" ? "errored" : state;
|
||||
observations.push({ ...observation, outcome });
|
||||
}
|
||||
}
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
|
||||
export default class EvalHarnessReporter implements Reporter {
|
||||
private vitest: Vitest | undefined;
|
||||
|
||||
onInit(vitest: Vitest): void {
|
||||
this.vitest = vitest;
|
||||
}
|
||||
|
||||
async onTestCaseResult(test: TestCase): Promise<void> {
|
||||
await appendHarnessRunReport(test);
|
||||
}
|
||||
|
||||
onTestRunEnd(
|
||||
modules: ReadonlyArray<TestModule>,
|
||||
_errors: ReadonlyArray<SerializedError>,
|
||||
reason: TestRunEndReason,
|
||||
): void {
|
||||
if (reason === "interrupted") {
|
||||
this.vitest?.logger.log("\nEval comparisons unavailable: test run interrupted.");
|
||||
return;
|
||||
}
|
||||
const report = summarizeHarnessComparisons(collectHarnessObservations(modules));
|
||||
const formatted = formatHarnessComparisonReport(report);
|
||||
if (formatted) this.vitest?.logger.log(`\n${formatted}`);
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
import { afterEach } from "vitest";
|
||||
import type {} from "vitest-evals";
|
||||
import { recordEvalSessionArtifact } from "./artifacts.ts";
|
||||
|
||||
afterEach(async ({ task }) => {
|
||||
const run = task.meta.harness?.run;
|
||||
if (run) await recordEvalSessionArtifact(task, run);
|
||||
});
|
||||
@@ -1,438 +0,0 @@
|
||||
import { styleText } from "node:util";
|
||||
|
||||
type HarnessObservationOutcome = "scored" | "unscored" | "skipped" | "pending" | "errored";
|
||||
|
||||
type HarnessObservationBase = {
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
testName: string;
|
||||
file: string;
|
||||
harness: string;
|
||||
baseline: string;
|
||||
candidates: string[];
|
||||
repetition: number;
|
||||
totalTokens?: number;
|
||||
totalMs?: number;
|
||||
estimatedCostUsd?: number;
|
||||
};
|
||||
|
||||
export type HarnessObservation = HarnessObservationBase &
|
||||
({ outcome: "scored"; score: number } | { outcome: Exclude<HarnessObservationOutcome, "scored">; score?: never });
|
||||
|
||||
export type PairedMetricSummary = {
|
||||
totalPairs: number;
|
||||
eligiblePairs: number;
|
||||
baselineMean: number | null;
|
||||
candidateMean: number | null;
|
||||
meanDelta: number | null;
|
||||
};
|
||||
|
||||
export type CorrectnessLiftSummary = {
|
||||
totalPairs: number;
|
||||
eligiblePairs: number;
|
||||
baselinePassRate: number | null;
|
||||
candidatePassRate: number | null;
|
||||
lift: number | null;
|
||||
baselineWins: number;
|
||||
candidateWins: number;
|
||||
ties: number;
|
||||
};
|
||||
|
||||
export type HarnessPairComparison = {
|
||||
baseline: string;
|
||||
candidate: string;
|
||||
correctness: CorrectnessLiftSummary;
|
||||
totalTokens: PairedMetricSummary;
|
||||
totalMs: PairedMetricSummary;
|
||||
estimatedCostUsd: PairedMetricSummary;
|
||||
};
|
||||
|
||||
export type HarnessComparisonDiagnostic = {
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
testName: string;
|
||||
file: string;
|
||||
repetition: number;
|
||||
harness: string;
|
||||
reason: "missing-observation" | "duplicate-observation" | "harness-error" | "missing-score" | "unscorable-outcome";
|
||||
};
|
||||
|
||||
export type HarnessEvalSetReport = {
|
||||
evalSet: string;
|
||||
comparisons: HarnessPairComparison[];
|
||||
};
|
||||
|
||||
export type HarnessComparisonReport = {
|
||||
schemaVersion: 1;
|
||||
evalSets: HarnessEvalSetReport[];
|
||||
diagnostics: HarnessComparisonDiagnostic[];
|
||||
};
|
||||
|
||||
type HarnessDescriptor = {
|
||||
name: string;
|
||||
index: number;
|
||||
};
|
||||
|
||||
type ObservationGroup = {
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
testName: string;
|
||||
file: string;
|
||||
repetition: number;
|
||||
observationsByHarness: Map<string, HarnessObservation[]>;
|
||||
};
|
||||
|
||||
type EvalSetData = {
|
||||
baseline: HarnessDescriptor;
|
||||
candidatesByName: Map<string, HarnessDescriptor>;
|
||||
groupsByKey: Map<string, ObservationGroup>;
|
||||
};
|
||||
|
||||
type ObservationPair = {
|
||||
baseline: HarnessObservation;
|
||||
candidate: HarnessObservation;
|
||||
};
|
||||
|
||||
function getOrCreate<K, V extends object>(map: Map<K, V>, key: K, create: () => V): V {
|
||||
const existing = map.get(key);
|
||||
if (existing !== undefined) return existing;
|
||||
const value = create();
|
||||
map.set(key, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
function mean(values: readonly number[]): number | null {
|
||||
return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
function preciseDifference(left: number, right: number): number {
|
||||
return Number((left - right).toPrecision(15));
|
||||
}
|
||||
|
||||
function groupObservations(observations: readonly HarnessObservation[]): Map<string, EvalSetData> {
|
||||
const evalSets = new Map<string, EvalSetData>();
|
||||
for (const observation of observations) {
|
||||
const evalSet = getOrCreate(evalSets, observation.evalSet, () => ({
|
||||
baseline: { name: observation.baseline, index: 0 },
|
||||
candidatesByName: new Map(),
|
||||
groupsByKey: new Map(),
|
||||
}));
|
||||
|
||||
for (const [index, name] of observation.candidates.entries()) {
|
||||
const existing = evalSet.candidatesByName.get(name);
|
||||
if (!existing || index < existing.index) evalSet.candidatesByName.set(name, { name, index });
|
||||
}
|
||||
|
||||
const group = getOrCreate(
|
||||
evalSet.groupsByKey,
|
||||
JSON.stringify([observation.file, observation.testName, observation.groupKey]),
|
||||
() => ({
|
||||
evalSet: observation.evalSet,
|
||||
groupKey: observation.groupKey,
|
||||
testName: observation.testName,
|
||||
file: observation.file,
|
||||
repetition: observation.repetition,
|
||||
observationsByHarness: new Map(),
|
||||
}),
|
||||
);
|
||||
getOrCreate(group.observationsByHarness, observation.harness, (): HarnessObservation[] => []).push(observation);
|
||||
}
|
||||
return evalSets;
|
||||
}
|
||||
|
||||
function orderedHarnesses(evalSet: EvalSetData): HarnessDescriptor[] {
|
||||
return [
|
||||
evalSet.baseline,
|
||||
...[...evalSet.candidatesByName.values()].sort(
|
||||
(left, right) => left.index - right.index || left.name.localeCompare(right.name),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
function orderedCandidates(evalSet: EvalSetData): HarnessDescriptor[] {
|
||||
return [...evalSet.candidatesByName.values()].sort(
|
||||
(left, right) => left.index - right.index || left.name.localeCompare(right.name),
|
||||
);
|
||||
}
|
||||
|
||||
function orderedGroups(evalSet: EvalSetData): ObservationGroup[] {
|
||||
return [...evalSet.groupsByKey.values()].sort(
|
||||
(left, right) => left.groupKey.localeCompare(right.groupKey) || left.repetition - right.repetition,
|
||||
);
|
||||
}
|
||||
|
||||
function collectDiagnostics(
|
||||
harnesses: readonly HarnessDescriptor[],
|
||||
groups: readonly ObservationGroup[],
|
||||
): HarnessComparisonDiagnostic[] {
|
||||
const diagnostics: HarnessComparisonDiagnostic[] = [];
|
||||
for (const group of groups) {
|
||||
for (const { name: harness } of harnesses) {
|
||||
const observations = group.observationsByHarness.get(harness) ?? [];
|
||||
let reason: HarnessComparisonDiagnostic["reason"] | undefined;
|
||||
if (observations.length === 0) reason = "missing-observation";
|
||||
else if (observations.length > 1) reason = "duplicate-observation";
|
||||
else if (observations[0].outcome === "errored") reason = "harness-error";
|
||||
else if (observations[0].outcome === "unscored") {
|
||||
reason = "missing-score";
|
||||
} else if (observations[0].outcome !== "scored") {
|
||||
reason = "unscorable-outcome";
|
||||
}
|
||||
if (!reason) continue;
|
||||
diagnostics.push({
|
||||
evalSet: group.evalSet,
|
||||
groupKey: group.groupKey,
|
||||
testName: group.testName,
|
||||
file: group.file,
|
||||
repetition: group.repetition,
|
||||
harness,
|
||||
reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
return diagnostics;
|
||||
}
|
||||
|
||||
function pairObservations(
|
||||
groups: readonly ObservationGroup[],
|
||||
baselineHarness: string,
|
||||
candidateHarness: string,
|
||||
): ObservationPair[] {
|
||||
const pairs: ObservationPair[] = [];
|
||||
for (const group of groups) {
|
||||
const baseline = group.observationsByHarness.get(baselineHarness) ?? [];
|
||||
const candidate = group.observationsByHarness.get(candidateHarness) ?? [];
|
||||
if (baseline.length === 1 && candidate.length === 1) {
|
||||
pairs.push({ baseline: baseline[0], candidate: candidate[0] });
|
||||
}
|
||||
}
|
||||
return pairs;
|
||||
}
|
||||
|
||||
function summarizeMetric(
|
||||
pairs: readonly ObservationPair[],
|
||||
select: (observation: HarnessObservation) => number | undefined,
|
||||
totalPairs: number,
|
||||
): PairedMetricSummary {
|
||||
const baselineValues: number[] = [];
|
||||
const candidateValues: number[] = [];
|
||||
for (const { baseline, candidate } of pairs) {
|
||||
if (baseline.outcome !== "scored" || candidate.outcome !== "scored") continue;
|
||||
const baselineValue = select(baseline);
|
||||
const candidateValue = select(candidate);
|
||||
if (
|
||||
baselineValue === undefined ||
|
||||
candidateValue === undefined ||
|
||||
!Number.isFinite(baselineValue) ||
|
||||
!Number.isFinite(candidateValue)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
baselineValues.push(baselineValue);
|
||||
candidateValues.push(candidateValue);
|
||||
}
|
||||
|
||||
const baselineMean = mean(baselineValues);
|
||||
const candidateMean = mean(candidateValues);
|
||||
return {
|
||||
totalPairs,
|
||||
eligiblePairs: baselineValues.length,
|
||||
baselineMean,
|
||||
candidateMean,
|
||||
meanDelta:
|
||||
baselineMean === null || candidateMean === null ? null : preciseDifference(candidateMean, baselineMean),
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeCorrectness(pairs: readonly ObservationPair[], totalPairs: number): CorrectnessLiftSummary {
|
||||
let eligiblePairs = 0;
|
||||
let baselinePasses = 0;
|
||||
let candidatePasses = 0;
|
||||
let baselineWins = 0;
|
||||
let candidateWins = 0;
|
||||
let ties = 0;
|
||||
|
||||
for (const { baseline, candidate } of pairs) {
|
||||
if (baseline.outcome !== "scored" || candidate.outcome !== "scored") continue;
|
||||
eligiblePairs += 1;
|
||||
const baselinePassed = baseline.score >= 1;
|
||||
const candidatePassed = candidate.score >= 1;
|
||||
if (baselinePassed) baselinePasses += 1;
|
||||
if (candidatePassed) candidatePasses += 1;
|
||||
if (baselinePassed === candidatePassed) ties += 1;
|
||||
else if (baselinePassed) baselineWins += 1;
|
||||
else candidateWins += 1;
|
||||
}
|
||||
|
||||
const baselinePassRate = eligiblePairs === 0 ? null : baselinePasses / eligiblePairs;
|
||||
const candidatePassRate = eligiblePairs === 0 ? null : candidatePasses / eligiblePairs;
|
||||
return {
|
||||
totalPairs,
|
||||
eligiblePairs,
|
||||
baselinePassRate,
|
||||
candidatePassRate,
|
||||
lift:
|
||||
baselinePassRate === null || candidatePassRate === null
|
||||
? null
|
||||
: preciseDifference(candidatePassRate, baselinePassRate),
|
||||
baselineWins,
|
||||
candidateWins,
|
||||
ties,
|
||||
};
|
||||
}
|
||||
|
||||
function compareHarnesses(
|
||||
baseline: HarnessDescriptor,
|
||||
candidate: HarnessDescriptor,
|
||||
groups: readonly ObservationGroup[],
|
||||
): HarnessPairComparison {
|
||||
const pairs = pairObservations(groups, baseline.name, candidate.name);
|
||||
return {
|
||||
baseline: baseline.name,
|
||||
candidate: candidate.name,
|
||||
correctness: summarizeCorrectness(pairs, groups.length),
|
||||
totalTokens: summarizeMetric(pairs, ({ totalTokens }) => totalTokens, groups.length),
|
||||
totalMs: summarizeMetric(pairs, ({ totalMs }) => totalMs, groups.length),
|
||||
estimatedCostUsd: summarizeMetric(pairs, ({ estimatedCostUsd }) => estimatedCostUsd, groups.length),
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeHarnessComparisons(observations: readonly HarnessObservation[]): HarnessComparisonReport {
|
||||
const evalSets: HarnessEvalSetReport[] = [];
|
||||
const diagnostics: HarnessComparisonDiagnostic[] = [];
|
||||
for (const [evalSet, data] of [...groupObservations(observations)].sort(([left], [right]) =>
|
||||
left.localeCompare(right),
|
||||
)) {
|
||||
const harnesses = orderedHarnesses(data);
|
||||
const candidates = orderedCandidates(data);
|
||||
const groups = orderedGroups(data);
|
||||
evalSets.push({
|
||||
evalSet,
|
||||
comparisons: candidates.map((candidate) => compareHarnesses(data.baseline, candidate, groups)),
|
||||
});
|
||||
diagnostics.push(...collectDiagnostics(harnesses, groups));
|
||||
}
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
evalSets,
|
||||
diagnostics: diagnostics.sort(
|
||||
(left, right) =>
|
||||
left.evalSet.localeCompare(right.evalSet) ||
|
||||
left.file.localeCompare(right.file) ||
|
||||
left.groupKey.localeCompare(right.groupKey) ||
|
||||
left.repetition - right.repetition ||
|
||||
left.harness.localeCompare(right.harness),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function formatPercentage(value: number | null): string {
|
||||
return value === null ? "unavailable" : `${(value * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function formatSigned(value: number, fractionDigits: number): string {
|
||||
return `${value >= 0 ? "+" : ""}${value.toFixed(fractionDigits)}`;
|
||||
}
|
||||
|
||||
function formatCoverage(eligiblePairs: number, totalPairs: number): string {
|
||||
return styleText("gray", `(${eligiblePairs}/${totalPairs} pairs)`);
|
||||
}
|
||||
|
||||
function formatReportLine(label: string, value: string): string {
|
||||
return ` ${styleText("gray", label.padStart(9))} ${value}`;
|
||||
}
|
||||
|
||||
function colorDelta(value: number, formatted: string, positiveIsBetter: boolean): string {
|
||||
if (value === 0) return styleText("gray", formatted);
|
||||
const improved = positiveIsBetter ? value > 0 : value < 0;
|
||||
return styleText(improved ? "green" : "red", formatted);
|
||||
}
|
||||
|
||||
function formatMetric(
|
||||
label: string,
|
||||
metric: PairedMetricSummary,
|
||||
formatValue: (value: number) => string,
|
||||
formatDelta: (value: number) => string,
|
||||
comparisonPairs: number,
|
||||
): string {
|
||||
const coverage =
|
||||
metric.eligiblePairs === 0 || metric.eligiblePairs === comparisonPairs
|
||||
? ""
|
||||
: ` ${formatCoverage(metric.eligiblePairs, metric.totalPairs)}`;
|
||||
if (metric.baselineMean === null || metric.candidateMean === null || metric.meanDelta === null) {
|
||||
return formatReportLine(label, `${styleText("yellow", "unavailable")}${coverage}`);
|
||||
}
|
||||
const delta = colorDelta(metric.meanDelta, formatDelta(metric.meanDelta), false);
|
||||
const values = styleText(
|
||||
"gray",
|
||||
`(candidate ${formatValue(metric.candidateMean)}, baseline ${formatValue(metric.baselineMean)})`,
|
||||
);
|
||||
return formatReportLine(label, `${delta} ${values}${coverage}`);
|
||||
}
|
||||
|
||||
export function formatHarnessComparisonReport(report: HarnessComparisonReport): string {
|
||||
if (report.evalSets.every(({ comparisons }) => comparisons.length === 0)) return "";
|
||||
const lines = [styleText("bold", "Eval Comparisons")];
|
||||
for (const evalSet of report.evalSets) {
|
||||
lines.push(` ${evalSet.evalSet}`);
|
||||
for (const [index, comparison] of evalSet.comparisons.entries()) {
|
||||
if (index > 0) lines.push("");
|
||||
const { correctness } = comparison;
|
||||
lines.push(formatReportLine("Baseline", comparison.baseline));
|
||||
lines.push(
|
||||
formatReportLine(
|
||||
"Candidate",
|
||||
`${comparison.candidate} ${formatCoverage(correctness.eligiblePairs, correctness.totalPairs)}`,
|
||||
),
|
||||
);
|
||||
if (correctness.lift === null) {
|
||||
lines.push(formatReportLine("Pass rate", styleText("yellow", "unavailable")));
|
||||
} else {
|
||||
const lift = correctness.lift * 100;
|
||||
const delta = colorDelta(lift, `${formatSigned(lift, 1)} pp`, true);
|
||||
const values = styleText(
|
||||
"gray",
|
||||
`(candidate ${formatPercentage(correctness.candidatePassRate)}, baseline ${formatPercentage(correctness.baselinePassRate)})`,
|
||||
);
|
||||
lines.push(formatReportLine("Pass rate", `${delta} ${values}`));
|
||||
}
|
||||
lines.push(
|
||||
formatMetric(
|
||||
"Tokens",
|
||||
comparison.totalTokens,
|
||||
(value) => value.toFixed(1),
|
||||
(value) => formatSigned(value, 1),
|
||||
correctness.eligiblePairs,
|
||||
),
|
||||
);
|
||||
lines.push(
|
||||
formatMetric(
|
||||
"Latency",
|
||||
comparison.totalMs,
|
||||
(value) => `${value.toFixed(1)}ms`,
|
||||
(value) => `${formatSigned(value, 1)}ms`,
|
||||
correctness.eligiblePairs,
|
||||
),
|
||||
);
|
||||
lines.push(
|
||||
formatMetric(
|
||||
"Est. cost",
|
||||
comparison.estimatedCostUsd,
|
||||
(value) => `$${value.toFixed(4)}`,
|
||||
(value) => `${value >= 0 ? "+" : "-"}$${Math.abs(value).toFixed(4)}`,
|
||||
correctness.eligiblePairs,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (report.diagnostics.length > 0) {
|
||||
lines.push(` ${styleText("yellow", "Incomplete observations")}`);
|
||||
for (const diagnostic of report.diagnostics) {
|
||||
lines.push(
|
||||
` ${diagnostic.reason}: ${diagnostic.file}/${diagnostic.testName} repetition ${diagnostic.repetition}, harness ${diagnostic.harness}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
Reference in New Issue
Block a user