chore(vendor): import pi 1.3.0

Imported as a linear vendor commit so forge rebases do not replay the upstream root tree.

git-subtree-dir: third_party/pi

git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0

third_party/pi/packages/evals/evals/acme-server.ts

@@ -0,0 +1,154 @@
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
+
+export const OPENAI_PROVIDER_ID = "acme";
+export const OPENAI_MODEL_ID = "acme-chat";
+export const OPENAI_PROBE_PROMPT = "Reply with ACME_OK.";
+export const OPENAI_PROBE_RESPONSE = "ACME_OK";
+export const STREAM_PROVIDER_ID = "acme-stream";
+export const STREAM_MODEL_ID = "acme-stream-chat";
+export const STREAM_PROBE_PROMPT = "Reply with ACME_STREAM_OK.";
+export const STREAM_PROBE_RESPONSE = "ACME_STREAM_OK";
+
+export const STREAM_API_DOCUMENTATION = `${JSON.stringify(
+	{
+		name: "Acme Streaming API",
+		request: {
+			method: "POST",
+			path: "/generate",
+			headers: { "content-type": "application/json", "x-acme-key": "resolved credential" },
+			body: { model: STREAM_MODEL_ID, messages: [{ role: "user", content: "Hello" }], stream: true },
+		},
+		response: {
+			contentType: "application/x-ndjson",
+			events: [
+				{ type: "text_delta", text: "Hello" },
+				{ type: "usage", input_tokens: 3, output_tokens: 2 },
+				{ type: "done", reason: "stop" },
+			],
+		},
+	},
+	null,
+	2,
+)}\n`;
+
+type ServerMode = "openai" | "stream";
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function rejectRequest(response: ServerResponse, status: number, message: string): void {
+	response.writeHead(status, { "content-type": "text/plain" });
+	response.end(message);
+}
+
+async function readJson(request: IncomingMessage): Promise<unknown> {
+	let body = "";
+	for await (const chunk of request) body += chunk.toString();
+	return JSON.parse(body);
+}
+
+export type AcmeServer = {
+	start(): Promise<void>;
+	stop(): Promise<void>;
+	reset(): void;
+	origin(): string;
+	baseUrl(): string;
+	validRequestReceived(): boolean;
+};
+
+export function createAcmeServer(mode: ServerMode): AcmeServer {
+	let server: Server | undefined;
+	let serverOrigin = "";
+	let receivedValidRequest = false;
+
+	async function handle(request: IncomingMessage, response: ServerResponse): Promise<void> {
+		const expectedPath = mode === "openai" ? "/v1/chat/completions" : "/generate";
+		if (request.url !== expectedPath) return rejectRequest(response, 404, "Unknown endpoint");
+		if (request.method !== "POST") return rejectRequest(response, 405, "Expected POST");
+		if (!request.headers["content-type"]?.startsWith("application/json")) {
+			return rejectRequest(response, 415, "Expected application/json");
+		}
+		let payload: unknown;
+		try {
+			payload = await readJson(request);
+		} catch {
+			return rejectRequest(response, 400, "Invalid JSON");
+		}
+		if (!isRecord(payload)) return rejectRequest(response, 422, "Expected a JSON object");
+		const messages = Array.isArray(payload.messages) ? payload.messages : [];
+		const userMessage = messages.find(
+			(message): message is Record<string, unknown> => isRecord(message) && message.role === "user",
+		);
+		const prompt = typeof userMessage?.content === "string" ? userMessage.content : undefined;
+
+		if (mode === "stream") {
+			if (request.headers["x-acme-key"] !== "resolved-stream-key") {
+				return rejectRequest(response, 401, "Invalid Acme Stream credential");
+			}
+			if (payload.model !== STREAM_MODEL_ID || prompt === undefined || payload.stream !== true) {
+				return rejectRequest(response, 422, "Invalid Acme Stream request");
+			}
+			receivedValidRequest = prompt === STREAM_PROBE_PROMPT;
+			response.writeHead(200, { "content-type": "application/x-ndjson" });
+			response.write(`${JSON.stringify({ type: "text_delta", text: "ACME_" })}\n`);
+			response.write(`${JSON.stringify({ type: "text_delta", text: "STREAM_OK" })}\n`);
+			response.write(`${JSON.stringify({ type: "usage", input_tokens: 4, output_tokens: 3 })}\n`);
+			response.end(`${JSON.stringify({ type: "done", reason: "stop" })}\n`);
+			return;
+		}
+
+		if (request.headers.authorization !== "Bearer resolved-acme-key") {
+			return rejectRequest(response, 401, "Invalid Acme credential");
+		}
+		if (payload.model !== OPENAI_MODEL_ID || prompt === undefined || payload.stream !== true) {
+			return rejectRequest(response, 422, "Invalid OpenAI-compatible request");
+		}
+		receivedValidRequest = prompt === OPENAI_PROBE_PROMPT;
+		response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" });
+		response.write(
+			`data: ${JSON.stringify({
+				id: "chatcmpl-acme",
+				object: "chat.completion.chunk",
+				created: 0,
+				model: OPENAI_MODEL_ID,
+				choices: [{ index: 0, delta: { role: "assistant", content: OPENAI_PROBE_RESPONSE }, finish_reason: null }],
+			})}\n\n`,
+		);
+		response.write(
+			`data: ${JSON.stringify({
+				id: "chatcmpl-acme",
+				object: "chat.completion.chunk",
+				created: 0,
+				model: OPENAI_MODEL_ID,
+				choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
+				usage: { prompt_tokens: 3, completion_tokens: 2 },
+			})}\n\n`,
+		);
+		response.end("data: [DONE]\n\n");
+	}
+
+	return {
+		async start() {
+			server = createServer((request, response) => void handle(request, response));
+			await new Promise<void>((resolve, reject) => {
+				server?.once("error", reject);
+				server?.listen(0, "127.0.0.1", resolve);
+			});
+			const address = server.address();
+			if (!address || typeof address === "string") throw new Error("Acme fixture server did not bind a TCP port.");
+			serverOrigin = `http://127.0.0.1:${address.port}`;
+		},
+		async stop() {
+			if (!server) return;
+			await new Promise<void>((resolve, reject) => server?.close((error) => (error ? reject(error) : resolve())));
+			server = undefined;
+		},
+		reset() {
+			receivedValidRequest = false;
+		},
+		origin: () => serverOrigin,
+		baseUrl: () => `${serverOrigin}/v1`,
+		validRequestReceived: () => receivedValidRequest,
+	};
+}

third_party/pi/packages/evals/evals/configured-runtime.ts

@@ -0,0 +1,120 @@
+import { join } from "node:path";
+import { type Api, type Context, contentText, type Model, type ModelsSimpleStreamOptions } from "@earendil-works/pi-ai";
+import { ModelRuntime } from "@earendil-works/pi-coding-agent";
+
+type ModelFields = {
+	id: string;
+	name: string;
+	provider: string;
+	reasoning: boolean;
+	input: Array<"text" | "image">;
+	cost: { input: number; output: number; cacheRead: number; cacheWrite: number };
+	contextWindow: number;
+	maxTokens: number;
+};
+
+export type ProviderRuntimeOutput = {
+	result:
+		| {
+				validRequestReceived: boolean;
+				model: ModelFields;
+				response: { text: string; stopReason: string; inputTokens: number; outputTokens: number };
+		  }
+		| { error: string };
+};
+
+export type AddedModelOutput = {
+	result: { model: ModelFields; existingModelsPreserved: boolean } | { error: string };
+};
+
+export type ProviderScenario = {
+	providerId: string;
+	modelId: string;
+	createContext(): Context;
+	options?: ModelsSimpleStreamOptions;
+	validRequestReceived(): boolean;
+};
+
+function modelFields(model: Model<Api>): ModelFields {
+	return {
+		id: model.id,
+		name: model.name,
+		provider: model.provider,
+		reasoning: model.reasoning,
+		input: [...model.input],
+		cost: {
+			input: model.cost.input,
+			output: model.cost.output,
+			cacheRead: model.cost.cacheRead,
+			cacheWrite: model.cost.cacheWrite,
+		},
+		contextWindow: model.contextWindow,
+		maxTokens: model.maxTokens,
+	};
+}
+
+export function loadConfiguredModelRuntime(agentDir: string): Promise<ModelRuntime> {
+	return ModelRuntime.create({
+		modelsPath: join(agentDir, "models.json"),
+		authPath: join(agentDir, "auth.json"),
+		modelsStorePath: join(agentDir, "models-store.json"),
+		allowModelNetwork: false,
+	});
+}
+
+export async function inspectProvider(
+	runtime: ModelRuntime,
+	scenario: ProviderScenario,
+): Promise<ProviderRuntimeOutput> {
+	await runtime.refresh({ allowNetwork: false });
+	const configurationError = runtime.getError();
+	if (configurationError) return { result: { error: configurationError } };
+	const model = runtime.getModel(scenario.providerId, scenario.modelId);
+	if (!model) {
+		return { result: { error: `Model ${scenario.providerId}/${scenario.modelId} is unavailable after reload.` } };
+	}
+	try {
+		const response = await runtime.completeSimple(model, scenario.createContext(), scenario.options);
+		return {
+			result: {
+				validRequestReceived: scenario.validRequestReceived(),
+				model: modelFields(model),
+				response: {
+					text: contentText(response.content),
+					stopReason: response.stopReason,
+					inputTokens: response.usage.input,
+					outputTokens: response.usage.output,
+				},
+			},
+		};
+	} catch (error) {
+		return { result: { error: error instanceof Error ? error.message : String(error) } };
+	}
+}
+
+export async function inspectAddedModel(
+	runtime: ModelRuntime,
+	providerId: string,
+	modelId: string,
+): Promise<AddedModelOutput> {
+	const pristineRuntime = await ModelRuntime.create({ modelsPath: null, allowModelNetwork: false });
+	const pristineProvider = pristineRuntime.getProvider(providerId);
+	if (!pristineProvider) return { result: { error: `Built-in provider ${providerId} is unavailable.` } };
+	const existingModelIds = pristineProvider.getModels().map(({ id }) => id);
+	if (existingModelIds.length === 0) {
+		return { result: { error: `Built-in provider ${providerId} has no models.` } };
+	}
+	await runtime.refresh({ allowNetwork: false });
+	const configurationError = runtime.getError();
+	if (configurationError) return { result: { error: configurationError } };
+	const model = runtime.getModel(providerId, modelId);
+	if (!model) {
+		return { result: { error: `Model ${providerId}/${modelId} is unavailable after reload.` } };
+	}
+	return {
+		result: {
+			model: modelFields(model),
+			existingModelsPreserved: existingModelIds.every((id) => runtime.getModel(providerId, id) !== undefined),
+		},
+	};
+}

third_party/pi/packages/evals/evals/custom-provider.docs.eval.ts

@@ -0,0 +1,72 @@
+import { afterAll, beforeAll, beforeEach } from "vitest";
+import { describeEval, StructuredOutputJudge } from "vitest-evals";
+import {
+	createAcmeServer,
+	STREAM_API_DOCUMENTATION,
+	STREAM_MODEL_ID,
+	STREAM_PROBE_PROMPT,
+	STREAM_PROBE_RESPONSE,
+	STREAM_PROVIDER_ID,
+} from "./acme-server.ts";
+import { inspectProvider, type ProviderRuntimeOutput } from "./configured-runtime.ts";
+import { createPiDocumentationEvalHarness } from "../src/harness.ts";
+
+const DOCUMENTATION_PATH = "fixtures/acme-stream-api.json";
+const STREAM_API_KEY = "resolved-stream-key";
+const server = createAcmeServer("stream");
+
+beforeAll(() => {
+	process.env.ACME_STREAM_API_KEY = STREAM_API_KEY;
+	return server.start();
+});
+beforeEach(() => server.reset());
+afterAll(() => {
+	delete process.env.ACME_STREAM_API_KEY;
+	return server.stop();
+});
+
+const expected: ProviderRuntimeOutput = {
+	result: {
+		validRequestReceived: true,
+		model: {
+			id: STREAM_MODEL_ID,
+			name: "Acme Stream Chat",
+			provider: STREAM_PROVIDER_ID,
+			reasoning: false,
+			input: ["text"],
+			cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+			contextWindow: 16384,
+			maxTokens: 2048,
+		},
+		response: { text: STREAM_PROBE_RESPONSE, stopReason: "stop", inputTokens: 4, outputTokens: 3 },
+	},
+};
+
+const harness = createPiDocumentationEvalHarness({
+	workspaceFiles: { [DOCUMENTATION_PATH]: STREAM_API_DOCUMENTATION },
+	output: ({ session }) =>
+		inspectProvider(session.modelRuntime, {
+			providerId: STREAM_PROVIDER_ID,
+			modelId: STREAM_MODEL_ID,
+			createContext: () => ({
+				messages: [{ role: "user", content: STREAM_PROBE_PROMPT, timestamp: 0 }],
+			}),
+			options: { env: { ACME_STREAM_API_KEY: STREAM_API_KEY }, maxTokens: 32 },
+			validRequestReceived: server.validRequestReceived,
+		}),
+});
+const judge = StructuredOutputJudge({ expected, match: "strict", allowExtras: false });
+
+describeEval("Add custom streaming provider", { harness, judges: [judge], judgeThreshold: null }, (it) => {
+	it("implements a provider from an API fixture", async ({ run }) => {
+		await run([
+			{
+				type: "prompt",
+				content: `Configure this running Pi installation with Acme Stream as a provider. Do not create a project package or modify project source. Its provider ID is ${STREAM_PROVIDER_ID}, its API is at ${server.origin()}, and its API documentation is in ./${DOCUMENTATION_PATH}. Read its credential from the ACME_STREAM_API_KEY environment variable.
+
+It offers one model, ${STREAM_MODEL_ID}, shown as “Acme Stream Chat”. The model accepts text, does not support reasoning, has a 16,384-token context window and a 2,048-token maximum output, and has no usage cost.`,
+			},
+			{ type: "reload" },
+		]);
+	});
+});

third_party/pi/packages/evals/evals/documentation-audit.eval.ts

@@ -0,0 +1,66 @@
+import { globSync } from "node:fs";
+import { resolve } from "node:path";
+import { Type } from "@earendil-works/pi-ai";
+import { defineTool } from "@earendil-works/pi-coding-agent";
+import { expect } from "vitest";
+import { describeEval, toolCalls } from "vitest-evals";
+import { createPiCodingAgentHarness } from "../src/harness.ts";
+
+const TOOL_NAME = "submit_documentation_audit";
+const submitAudit = defineTool({
+	name: TOOL_NAME,
+	label: "Submit documentation audit",
+	description: "Submit the final verdict after completing the documentation investigation.",
+	promptSnippet: "Submit the final documentation audit as validated structured data",
+	parameters: Type.Object(
+		{
+			verdict: Type.Union([Type.Literal("match"), Type.Literal("mismatch"), Type.Literal("inconclusive")]),
+			explanation: Type.String({ minLength: 1, maxLength: 2000 }),
+		},
+		{ additionalProperties: false },
+	),
+	constrainedSampling: { type: "json_schema", strict: "prefer" },
+	async execute(_toolCallId, params) {
+		return {
+			content: [{ type: "text", text: "Documentation audit submitted." }],
+			details: params,
+			terminate: true,
+		};
+	},
+});
+
+const repositoryRoot = resolve(import.meta.dirname, "../../..");
+const docsRoot = resolve(repositoryRoot, "packages/coding-agent/docs");
+const pages = globSync("**/*.md", { cwd: docsRoot })
+	.map((path) => ({ path: path.replaceAll("\\", "/") }))
+	.sort((left, right) => left.path.localeCompare(right.path));
+const harness = createPiCodingAgentHarness({
+	name: "documentation-page-audit",
+	tools: ["read", "grep", "find", "ls", TOOL_NAME],
+	customTools: [submitAudit],
+});
+
+describeEval("Audit documentation against implementation", { harness }, (it) => {
+	it.for(pages)("$path matches the implementation", async ({ path }, { run }) => {
+		const documentationPath = resolve(docsRoot, path);
+		const result = await run(`Audit this Pi documentation page against the repository implementation.
+
+Documentation page: ${documentationPath}
+Repository root: ${repositoryRoot}
+
+Read the complete page and the relevant implementation. Report a mismatch only for a clear, user-visible contradiction between an explicit documentation claim and actual runtime behavior. Follow the runtime path; names, comments, types, isolated helpers, and tests are not sufficient evidence by themselves.
+
+Missing internal detail, ambiguous wording, hypothetical misuse, and undocumented edge cases are not mismatches. If the evidence is not decisive, report inconclusive. Otherwise report match.
+
+For a mismatch, quote the claim, cite the implementation path and symbol, and state the concrete behavior a user would observe.
+
+Call ${TOOL_NAME} exactly once as your final action. Do not return prose.`);
+
+		const auditCalls = toolCalls(result).filter((call) => call.name === TOOL_NAME);
+		expect(auditCalls).toHaveLength(1);
+		const auditCall = auditCalls[0];
+		expect(auditCall?.status).toBe("ok");
+		const explanation = auditCall?.arguments?.explanation;
+		expect(auditCall?.arguments?.verdict, typeof explanation === "string" ? explanation : undefined).toBe("match");
+	});
+});

third_party/pi/packages/evals/evals/extensions.docs.eval.ts

@@ -0,0 +1,60 @@
+import { contentText } from "@earendil-works/pi-ai";
+import { describeEval, StructuredOutputJudge, ToolCallJudge } from "vitest-evals";
+import { createPiDocumentationEvalHarness, DOCUMENTATION_EVAL_TOOLS } from "../src/harness.ts";
+
+const TOOL_NAME = "hello";
+const TOOL_ARGUMENTS = { name: "Bob" };
+const TOOL_RESULT = "Hello, Bob!";
+
+const harness = createPiDocumentationEvalHarness({
+	tools: [...DOCUMENTATION_EVAL_TOOLS, TOOL_NAME],
+	output: ({ response, session }) => {
+		const extensions = session.resourceLoader.getExtensions();
+		const result = [...session.messages]
+			.reverse()
+			.find((message) => message.role === "toolResult" && message.toolName === TOOL_NAME && !message.isError);
+		return {
+			response,
+			extensionErrors: extensions.errors,
+			toolLoaded: extensions.extensions.some(({ tools }) => tools.has(TOOL_NAME)),
+			toolResult: result?.role === "toolResult" ? contentText(result.content) : null,
+		};
+	},
+});
+
+describeEval(
+	"Create and use a tool extension",
+	{
+		harness,
+		judges: [
+			StructuredOutputJudge({
+				expected: {
+					response: TOOL_RESULT,
+					extensionErrors: [],
+					toolLoaded: true,
+					toolResult: TOOL_RESULT,
+				},
+				match: "strict",
+				allowExtras: false,
+			}),
+			ToolCallJudge({ expectedTools: [{ name: TOOL_NAME, arguments: TOOL_ARGUMENTS }] }),
+		],
+		judgeThreshold: null,
+	},
+	(it) => {
+		it("creates, reloads, and invokes the extension", async ({ run }) => {
+			await run([
+				{
+					type: "prompt",
+					content:
+						"Configure this running Pi installation with an extension containing a hello tool that takes a name and returns a greeting. Do not create project source. For example, passing Bob should return `Hello, Bob!`.",
+				},
+				{ type: "reload" },
+				{
+					type: "prompt",
+					content: "Use the hello tool to greet Bob. Respond with exactly the tool's greeting and nothing else.",
+				},
+			]);
+		});
+	},
+);

third_party/pi/packages/evals/evals/models.docs.eval.ts

@@ -0,0 +1,40 @@
+import { describeEval, StructuredOutputJudge } from "vitest-evals";
+import { inspectAddedModel, type AddedModelOutput } from "./configured-runtime.ts";
+import { createPiDocumentationEvalHarness } from "../src/harness.ts";
+
+const PROVIDER_ID = "openai";
+const MODEL_ID = "fixture-chat";
+const MODEL_NAME = "Fixture Chat";
+
+const expected: AddedModelOutput = {
+	result: {
+		model: {
+			id: MODEL_ID,
+			name: MODEL_NAME,
+			provider: PROVIDER_ID,
+			reasoning: true,
+			input: ["text"],
+			cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+			contextWindow: 32768,
+			maxTokens: 4096,
+		},
+		existingModelsPreserved: true,
+	},
+};
+
+const harness = createPiDocumentationEvalHarness({
+	output: ({ session }) => inspectAddedModel(session.modelRuntime, PROVIDER_ID, MODEL_ID),
+});
+const judge = StructuredOutputJudge({ expected, match: "strict", allowExtras: false });
+
+describeEval("Add model to existing provider", { harness, judges: [judge], judgeThreshold: null }, (it) => {
+	it("adds the model without replacing existing models", async ({ run }) => {
+		await run([
+			{
+				type: "prompt",
+				content: `Configure this running Pi installation with a new \`${PROVIDER_ID}/${MODEL_ID}\` model. Do not create project-local configuration. Show it as “${MODEL_NAME}”. It accepts text, supports reasoning, has a 32,768-token context window and a 4,096-token maximum output, and has no usage cost.`,
+			},
+			{ type: "reload" },
+		]);
+	});
+});

third_party/pi/packages/evals/evals/openai-provider.docs.eval.ts

@@ -0,0 +1,62 @@
+import { afterAll, beforeAll, beforeEach } from "vitest";
+import { describeEval, StructuredOutputJudge } from "vitest-evals";
+import {
+	createAcmeServer,
+	OPENAI_MODEL_ID,
+	OPENAI_PROBE_PROMPT,
+	OPENAI_PROBE_RESPONSE,
+	OPENAI_PROVIDER_ID,
+} from "./acme-server.ts";
+import { inspectProvider, type ProviderRuntimeOutput } from "./configured-runtime.ts";
+import { createPiDocumentationEvalHarness } from "../src/harness.ts";
+
+const server = createAcmeServer("openai");
+
+beforeAll(() => server.start());
+beforeEach(() => server.reset());
+afterAll(() => server.stop());
+
+const expected: ProviderRuntimeOutput = {
+	result: {
+		validRequestReceived: true,
+		model: {
+			id: OPENAI_MODEL_ID,
+			name: "Acme Chat",
+			provider: OPENAI_PROVIDER_ID,
+			reasoning: false,
+			input: ["text"],
+			cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+			contextWindow: 32768,
+			maxTokens: 4096,
+		},
+		response: { text: OPENAI_PROBE_RESPONSE, stopReason: "stop", inputTokens: 3, outputTokens: 2 },
+	},
+};
+
+const harness = createPiDocumentationEvalHarness({
+	output: ({ session }) =>
+		inspectProvider(session.modelRuntime, {
+			providerId: OPENAI_PROVIDER_ID,
+			modelId: OPENAI_MODEL_ID,
+			createContext: () => ({
+				messages: [{ role: "user", content: OPENAI_PROBE_PROMPT, timestamp: 0 }],
+			}),
+			options: { env: { ACME_API_KEY: "resolved-acme-key" }, maxTokens: 32 },
+			validRequestReceived: server.validRequestReceived,
+		}),
+});
+const judge = StructuredOutputJudge({ expected, match: "strict", allowExtras: false });
+
+describeEval("Add OpenAI-compatible provider", { harness, judges: [judge], judgeThreshold: null }, (it) => {
+	it("configures a provider that works through Pi", async ({ run }) => {
+		await run([
+			{
+				type: "prompt",
+				content: `Configure this running Pi installation with Acme as a provider. Do not create project-local configuration. Its provider ID is ${OPENAI_PROVIDER_ID}, its API is at ${server.baseUrl()}, and it uses OpenAI Chat Completions. Read its API key from the ACME_API_KEY environment variable.
+
+The provider offers one model, ${OPENAI_MODEL_ID}, shown as “Acme Chat”. It accepts text, does not support reasoning, has a 32,768-token context window and a 4,096-token maximum output, and has no usage cost.`,
+			},
+			{ type: "reload" },
+		]);
+	});
+});

third_party/pi/packages/evals/evals/smoke.eval.ts

@@ -0,0 +1,18 @@
+import { expect } from "vitest";
+import { describeEval } from "vitest-evals";
+import { createPiCodingAgentHarness } from "../src/harness.ts";
+
+const harness = createPiCodingAgentHarness({ noTools: "all" });
+
+describeEval("Answer a basic prompt", { harness }, (it) => {
+	it("returns the expected answer", async ({ run }) => {
+		const result = await run("What's the capital of France? Respond with only the city name.");
+		expect(result.output.trim()).toBe("Paris");
+		expect(result.errors).toEqual([]);
+		expect(result.usage).toMatchObject({
+			provider: process.env.PI_PROVIDER,
+			model: process.env.PI_MODEL,
+		});
+		expect(result.usage.totalTokens).toBeGreaterThan(0);
+	});
+});

third_party/pi/packages/evals/evals/tui.docs.eval.ts

@@ -0,0 +1,264 @@
+import { setTimeout } from "node:timers/promises";
+import { stripVTControlCharacters } from "node:util";
+import type { AssistantMessage } from "@earendil-works/pi-ai";
+import {
+	type AgentSession,
+	AgentSessionRuntime,
+	type AgentSessionServices,
+	InteractiveMode,
+	SessionManager,
+	createAgentSessionFromServices,
+} from "@earendil-works/pi-coding-agent";
+import { Levenshtein } from "autoevals";
+import { createJudge, describeEval } from "vitest-evals";
+import { createPiDocumentationEvalHarness, type PiCodingAgentInput } from "../src/harness.ts";
+
+const CONTEXT_WINDOW = 272_000;
+const BUILT_IN_CONTEXT_PATTERN = /\d+(?:\.\d+)?%\/272k(?: \(auto\))?/;
+const CONTEXT_FIXTURES = [
+	{ percent: 42.2, expectedBar: "████░░░░░░ 42%" },
+	{ percent: 65, expectedBar: "███████░░░ 65%" },
+	{ percent: 120, expectedBar: "██████████ 100%" },
+] as const;
+
+class RecordingTerminal {
+	readonly columns = 100;
+	readonly rows = 30;
+	readonly kittyProtocolActive = true;
+	private writes: string[] = [];
+	private resizeHandler?: () => void;
+
+	start(_onInput: (data: string) => void, onResize: () => void): void {
+		this.resizeHandler = onResize;
+	}
+
+	stop(): void {
+		this.resizeHandler = undefined;
+	}
+
+	async drainInput(_maxMs?: number, _idleMs?: number): Promise<void> {}
+
+	write(data: string): void {
+		this.writes.push(data);
+	}
+
+	moveBy(_lines: number): void {}
+	hideCursor(): void {}
+	showCursor(): void {}
+	clearLine(): void {}
+	clearFromCursor(): void {}
+	clearScreen(): void {}
+	setTitle(_title: string): void {}
+	setProgress(_active: boolean): void {}
+
+	requestResize(): void {
+		this.resizeHandler?.();
+	}
+
+	async takeRenders(): Promise<string[]> {
+		await setTimeout(30);
+		const output = this.writes.join("");
+		this.writes = [];
+		const startMarker = "\x1b[?2026h";
+		const endMarker = "\x1b[?2026l";
+		const renders: string[] = [];
+		let offset = 0;
+		while (true) {
+			const start = output.indexOf(startMarker, offset);
+			if (start === -1) return renders;
+			const contentStart = start + startMarker.length;
+			const end = output.indexOf(endMarker, contentStart);
+			if (end === -1) throw new Error("TUI emitted an incomplete synchronized render.");
+			renders.push(stripVTControlCharacters(output.slice(contentStart, end)));
+			offset = end + endMarker.length;
+		}
+	}
+}
+
+type TuiFooterOutput = {
+	builtInStatusLine: string | null;
+	observations: Array<{
+		percent: number;
+		renderedOutput: string;
+		contextUsage: { tokens: number | null; contextWindow: number; percent: number | null } | null;
+	}>;
+	extensionErrors: string[];
+};
+
+function lines(output: string): string[] {
+	return output.split(/[\r\n]+/).filter((line) => line.trim().length > 0);
+}
+
+function createContextMessage(model: NonNullable<AgentSession["model"]>, percent: number): AssistantMessage {
+	const tokens = Math.round((model.contextWindow * percent) / 100);
+	return {
+		role: "assistant",
+		content: [{ type: "text", text: "Context usage fixture" }],
+		api: model.api,
+		provider: model.provider,
+		model: model.id,
+		usage: {
+			input: tokens,
+			output: 0,
+			cacheRead: 0,
+			cacheWrite: 0,
+			totalTokens: tokens,
+			cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+		},
+		stopReason: "stop",
+		timestamp: Date.now(),
+	};
+}
+
+async function inspectContextFooter(session: AgentSession, agentDir: string): Promise<TuiFooterOutput> {
+	const activeModel = session.model;
+	if (!activeModel) throw new Error("TUI footer oracle requires an active model.");
+	const model = { ...activeModel, contextWindow: CONTEXT_WINDOW };
+	const services: AgentSessionServices = {
+		cwd: session.sessionManager.getCwd(),
+		agentDir,
+		modelRuntime: session.modelRuntime,
+		settingsManager: session.settingsManager,
+		resourceLoader: session.resourceLoader,
+		diagnostics: [],
+	};
+	const oracleSession = (
+		await createAgentSessionFromServices({
+			services,
+			sessionManager: SessionManager.inMemory(services.cwd),
+			model,
+			thinkingLevel: "high",
+			noTools: "all",
+		})
+	).session;
+	const runtime = new AgentSessionRuntime(oracleSession, services, async () => {
+		throw new Error("The TUI footer oracle does not replace sessions.");
+	});
+	const terminal = new RecordingTerminal();
+	const mode = new InteractiveMode(runtime, { tuiMode: "regular", terminal });
+	const previousOffline = process.env.PI_OFFLINE;
+	process.env.PI_OFFLINE = "1";
+	try {
+		oracleSession.sessionManager.appendMessage({
+			...createContextMessage(model, 0),
+			usage: {
+				input: 106_000,
+				output: 5_800,
+				cacheRead: 606_000,
+				cacheWrite: 0,
+				totalTokens: 717_800,
+				cost: { input: 1.008, output: 0, cacheRead: 0, cacheWrite: 0, total: 1.008 },
+			},
+		});
+		let builtInStatusLine: string | null = null;
+		const observations: TuiFooterOutput["observations"] = [];
+		for (const [index, { percent }] of CONTEXT_FIXTURES.entries()) {
+			oracleSession.agent.state.messages = [createContextMessage(model, percent)];
+			if (index === 0) await mode.init();
+			else terminal.requestResize();
+			const renders = await terminal.takeRenders();
+			if (index === 0) {
+				builtInStatusLine = renders.flatMap(lines).find((line) => BUILT_IN_CONTEXT_PATTERN.test(line)) ?? null;
+			}
+			observations.push({
+				percent,
+				renderedOutput: renders.at(-1) ?? "",
+				contextUsage: oracleSession.getContextUsage() ?? null,
+			});
+		}
+		return {
+			builtInStatusLine,
+			observations,
+			extensionErrors: session.resourceLoader.getExtensions().errors.map(({ error }) => error),
+		};
+	} finally {
+		try {
+			mode.stop("resume-hint");
+		} finally {
+			try {
+				await runtime.dispose();
+			} finally {
+				if (previousOffline === undefined) delete process.env.PI_OFFLINE;
+				else process.env.PI_OFFLINE = previousOffline;
+			}
+		}
+	}
+}
+
+const harness = createPiDocumentationEvalHarness({
+	output: ({ session, agentDir }) => inspectContextFooter(session, agentDir),
+});
+
+const contextFooterJudge = createJudge<PiCodingAgentInput, TuiFooterOutput>(
+	"ContextFooterJudge",
+	async ({ output }) => {
+		let valid = output.extensionErrors.length === 0;
+		const similarities: number[] = [];
+		const contextMarker = "{context}";
+		const builtInStatusLine = output.builtInStatusLine?.trim().replace(/\s+/g, " ");
+		const reference = builtInStatusLine?.replace(BUILT_IN_CONTEXT_PATTERN, contextMarker);
+		if (!reference?.includes(contextMarker)) valid = false;
+		for (const [index, { percent, expectedBar }] of CONTEXT_FIXTURES.entries()) {
+			const observation = output.observations[index];
+			if (!observation || observation.percent !== percent) {
+				valid = false;
+				continue;
+			}
+			if (
+				observation.contextUsage?.contextWindow !== CONTEXT_WINDOW ||
+				Math.round((observation.contextUsage?.percent ?? -1) * 10) / 10 !== percent
+			) {
+				valid = false;
+			}
+			const actualStatusLine = lines(observation.renderedOutput).find(
+				(line) => line.includes(expectedBar) || BUILT_IN_CONTEXT_PATTERN.test(line),
+			);
+			if (!actualStatusLine) {
+				valid = false;
+				continue;
+			}
+			const normalizedStatusLine = actualStatusLine.trim().replace(/\s+/g, " ");
+			if (!normalizedStatusLine.includes(expectedBar) || BUILT_IN_CONTEXT_PATTERN.test(normalizedStatusLine)) {
+				valid = false;
+				continue;
+			}
+			for (const { expectedBar: otherBar } of CONTEXT_FIXTURES) {
+				if (otherBar !== expectedBar && normalizedStatusLine.includes(otherBar)) valid = false;
+			}
+			if (reference) {
+				const result = await Levenshtein({
+					expected: reference,
+					output: normalizedStatusLine.replace(expectedBar, contextMarker),
+				});
+				if (result.score !== null) similarities.push(result.score);
+			}
+		}
+		const retained = similarities.length === CONTEXT_FIXTURES.length ? Math.min(...similarities) : 0;
+		return {
+			score: valid ? 1 : 0,
+			metadata: {
+				rationale: valid
+					? `Footer retained ${Math.round(retained * 100)}% of the built-in status text.`
+					: "Context progress behavior was incorrect.",
+				retainedPercent: Math.round(retained * 100),
+			},
+		};
+	},
+);
+
+describeEval(
+	"Customize the interactive context footer",
+	{ harness, judges: [contextFooterJudge], judgeThreshold: null },
+	(it) => {
+		it("replaces numeric context usage with a progress bar", async ({ run }) => {
+			await run([
+				{
+					type: "prompt",
+					content:
+						"Change the Pi footer replacing the built-in context info, e.g. `42.2%/272k (auto)`, with a ten-cell progress bar to make it show ████░░░░░░ 42% instead. Clamp percentages to the 0-100 range. Just replace the context. Otherwise leave everything else the same.",
+				},
+				{ type: "reload" },
+			]);
+		});
+	},
+);