chore(vendor): import pi 1.3.0

Imported as a linear vendor commit so forge rebases do not replay the upstream root tree.

git-subtree-dir: third_party/pi

git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0

third_party/pi/packages/durable/src/codemode/describe.ts

@@ -0,0 +1,157 @@
+/**
+ * What the model reads about code mode, copied from the coding agent's code mode: the tool's fixed description (the
+ * script environment and its globals), and the `codemode` prompt section, which lists the tools scripts can call as
+ * TypeScript declarations, grouped by namespace and limited to a token budget. Tools that do not fit are found with
+ * `searchTools()`. The section renders from the conversation's callable tools, so it changes with them; the tool's
+ * description never does.
+ */
+import type { CodemodeJsonSchema } from "@earendil-works/pi-codemode";
+import { renderToolSample, toCodemodeIdentifier } from "@earendil-works/pi-codemode/declarations";
+import type { ToolRegistration } from "../harness/types.ts";
+import { MODELS_DESCRIPTION } from "./models.ts";
+
+/** A group of tools named by a common prefix, such as an MCP server's `mcp__github__`. */
+export type CodemodeNamespace = {
+	readonly name: string;
+	readonly description?: string;
+	readonly instructions?: string;
+	/** Tool names starting with it belong to the namespace; the longest matching prefix wins. */
+	readonly prefix: string;
+};
+
+/** Default token budget of the tool declarations in the `codemode` section, as in the coding agent. */
+export const DEFAULT_CODEMODE_INLINE_BUDGET = 3000;
+/** Characters per token when estimating the cost of a tool section. */
+const CHARS_PER_TOKEN = 4;
+
+const TEXT_OUTPUT_SCHEMA: CodemodeJsonSchema = { type: "string" };
+
+const DESCRIPTION_INTRO = `Run JavaScript that calls other tools. The input is raw JavaScript (not JSON, no code fence), run as an async function body in a QuickJS sandbox: top-level \`await\` and \`return\` work. No Node, file system, network, or timers.
+- \`await tools.<name>({ ...args })\` resolves to a string, or an object if the tool's declaration says so, and rejects with an Error on failure. A tool whose declaration returns an object resolves on failure too when it returned that object; check its fields. Calls still running when the script ends are cancelled.
+- The script survives a restart: it resumes where it was, without running finished calls again.
+- Optional first line: \`// @options: {"max_output_tokens": 10000, "timeout_ms": 60000}\``;
+
+/** The code mode tool's description: the script environment and its globals, the same for every conversation. */
+export function codemodeDescription(models: boolean): string {
+	const globals = [
+		"Globals:",
+		"- `text(value)`, `image(dataUrlOrImageBlock)`, `console.log(...)`, and top-level `return` add output; `exit()` ends the script. With several text items, each starts with a `==> text N/M <==` line, and `console` lines follow the other output in one `<console_output>` block. `image()` also saves the image to a temp file and the result names its path.",
+		"- `store(key, value)` and `load(key)` keep JSON values across codemode calls.",
+		"- `ALL_TOOLS`, `await searchTools(query, { limit?, namespace? })`, `await describeTool(name)`, `await describeNamespace(name)`: find tools the `codemode` section does not list.",
+	];
+	return [
+		DESCRIPTION_INTRO,
+		globals.join("\n"),
+		...(models ? [MODELS_DESCRIPTION] : []),
+		"The `codemode` section of the system prompt lists the tools scripts can call, by namespace, as far as space allows.",
+	].join("\n\n");
+}
+
+/** What a script sees of a tool: its declaration. Tools without an output schema resolve to their text output. */
+export function declarationOf(tool: ToolRegistration): {
+	name: string;
+	description: string;
+	inputSchema: CodemodeJsonSchema;
+	outputSchema: CodemodeJsonSchema;
+} {
+	return {
+		name: tool.name,
+		description: tool.description,
+		inputSchema: tool.parameters as CodemodeJsonSchema,
+		outputSchema: (tool.structuredOutputSchema as CodemodeJsonSchema | undefined) ?? TEXT_OUTPUT_SCHEMA,
+	};
+}
+
+/** `toolSample` of a tool: its description and TypeScript declaration. */
+export function sampleOf(tool: ToolRegistration): string {
+	return renderToolSample(declarationOf(tool));
+}
+
+/** The namespace of a tool: the one with the longest prefix its name starts with. */
+export function namespaceOf(name: string, namespaces: readonly CodemodeNamespace[]): CodemodeNamespace | undefined {
+	let found: CodemodeNamespace | undefined;
+	for (const namespace of namespaces) {
+		if (name.startsWith(namespace.prefix) && (found === undefined || namespace.prefix.length > found.prefix.length)) {
+			found = namespace;
+		}
+	}
+	return found;
+}
+
+/** `### \`id\` (\`raw name\`)` followed by the tool's description and declaration. */
+function renderToolSection(tool: ToolRegistration): string {
+	const id = toCodemodeIdentifier(tool.name);
+	const heading = id === tool.name ? `### \`${id}\`` : `### \`${id}\` (\`${tool.name}\`)`;
+	return `${heading}\n${sampleOf(tool).trim()}`;
+}
+
+type CatalogEntry = { name: string; section: string; cost: number };
+type CatalogGroup = { namespace: CodemodeNamespace | undefined; entries: CatalogEntry[] };
+
+/**
+ * Pick the tool sections that fit the budget: in each round every group (tools without a namespace first, then
+ * namespaces by name) places its cheapest remaining tool; a group whose next tool does not fit drops out while the
+ * others continue. Every namespace is represented before any namespace is complete.
+ */
+function selectCatalog(groups: readonly CatalogGroup[], budget: number): Set<string> {
+	const queues = groups.map((group) => [...group.entries].sort((a, b) => a.cost - b.cost));
+	const shown = new Set<string>();
+	let remaining = budget;
+	let active = queues.filter((queue) => queue.length > 0);
+	while (active.length > 0) {
+		active = active.filter((queue) => {
+			const next = queue[0]!;
+			if (next.cost > remaining) return false;
+			remaining -= next.cost;
+			shown.add(next.name);
+			queue.shift();
+			return queue.length > 0;
+		});
+	}
+	return shown;
+}
+
+/**
+ * The `codemode` section: the tools scripts can call, grouped by namespace, with their declarations as far as the
+ * budget allows. `undefined` without callable tools.
+ */
+export function codemodeSection(
+	tools: readonly ToolRegistration[],
+	namespaces: readonly CodemodeNamespace[],
+	budget: number,
+): string | undefined {
+	if (tools.length === 0) return undefined;
+	const groups = new Map<string, CatalogGroup>([["", { namespace: undefined, entries: [] }]]);
+	for (const tool of tools) {
+		const namespace = namespaceOf(tool.name, namespaces);
+		const key = namespace ? `ns:${namespace.name}` : "";
+		const group = groups.get(key) ?? { namespace, entries: [] };
+		groups.set(key, group);
+		const section = renderToolSection(tool);
+		group.entries.push({ name: tool.name, section, cost: Math.ceil(section.length / CHARS_PER_TOKEN) });
+	}
+	const ordered = [...groups.values()].sort((a, b) =>
+		a.namespace === undefined ? -1 : b.namespace === undefined ? 1 : a.namespace.name.localeCompare(b.namespace.name),
+	);
+	const shown = selectCatalog(ordered, budget);
+
+	const toolSections = ["Tools scripts can call (`await tools.<name>(args)`):"];
+	for (const { namespace, entries } of ordered) {
+		if (entries.length === 0) continue;
+		const visible = entries.filter((entry) => shown.has(entry.name));
+		if (namespace) {
+			const listing =
+				visible.length === entries.length
+					? ""
+					: visible.length === 0
+						? " (tools not listed)"
+						: " (some tools not listed)";
+			const description = namespace.description?.trim();
+			toolSections.push(`## ${namespace.name}${listing}${description ? `\n${description}` : ""}`);
+		} else if (visible.length < entries.length) {
+			toolSections.push("Some tools are not listed: find them with `searchTools()`.");
+		}
+		for (const entry of visible) toolSections.push(entry.section);
+	}
+	return toolSections.join("\n\n");
+}

third_party/pi/packages/durable/src/codemode/index.ts

@@ -0,0 +1,13 @@
+export {
+	CODEMODE_MODULE_VERSION,
+	CODEMODE_TOOL_NAME,
+	type CodemodeGlobal,
+	type CodemodeNestedCall,
+	CodemodeStoreDoc,
+	type CodemodeToolDetails,
+	type CodemodeToolInput,
+	type CodemodeToolOptions,
+	createCodemodeExtension,
+	createCodemodeTool,
+} from "./codemode.ts";
+export { type CodemodeNamespace, DEFAULT_CODEMODE_INLINE_BUDGET } from "./describe.ts";

third_party/pi/packages/durable/src/codemode/models.ts

@@ -0,0 +1,285 @@
+/**
+ * `models.*` for code mode scripts, copied from the coding agent's code mode: the model catalog, and classifier and
+ * image model calls.
+ *
+ * The catalog functions are host globals, journaled like any global. `models.classify()` and `models.generateImages()`
+ * are paid effects, so each runs as an internal tool (`callers: ["tools"]`, a reserved name, hidden from scripts'
+ * tool listings): a nested durable call with its own task, live row, abort, usage accounting, and replay policy. The
+ * code mode tool turns a script's call of the global into a call of that tool.
+ */
+import type { Context, JsonValue } from "@earendil-works/chord";
+import type {
+	AnyModel,
+	ClassifierContext,
+	ImagesContext,
+	Models,
+	ModelType,
+	ModelTypeMap,
+	Usage,
+} from "@earendil-works/pi-ai";
+import { Type } from "typebox";
+import { defineTool } from "../harness/define.ts";
+import type { ToolRegistration } from "../harness/types.ts";
+
+/** Reserved names of code mode's internal tools; they are never listed to scripts or the model. */
+export const CODEMODE_INTERNAL_PREFIX = "pi.codemode.";
+export const CLASSIFY_TOOL = `${CODEMODE_INTERNAL_PREFIX}models.classify`;
+export const GENERATE_IMAGES_TOOL = `${CODEMODE_INTERNAL_PREFIX}models.generateImages`;
+
+/** The global a script calls, by internal tool. */
+export const MODEL_CALLS: Readonly<Record<string, string>> = {
+	"models.classify": CLASSIFY_TOOL,
+	"models.generateImages": GENERATE_IMAGES_TOOL,
+};
+
+/** The catalog functions, host globals of the `models` namespace. */
+export const CATALOG_GLOBALS = [
+	"models.getModelsOfType",
+	"models.getAvailableOfType",
+	"models.getModelOfType",
+] as const;
+
+/** `models.classify()` and `models.generateImages()` calls one script may have in flight; the rest wait. */
+export const MAX_CONCURRENT_MODEL_CALLS = 4;
+
+const MODEL_TYPES: ReadonlySet<string> = new Set<ModelType>(["chat", "image", "classifier"]);
+
+export const MODELS_DESCRIPTION = [
+	"`models` (all functions return promises):",
+	'- `models.getModelsOfType(type, provider?)`, `models.getAvailableOfType(type, provider?)` (configured auth only), `models.getModelOfType(type, provider, id)`: the catalog; `type` is "chat", "image", or "classifier".',
+	'- `models.classify(model, { state: { ... }, images?: [{ type: "image", data, mimeType }], questions: { <id>: { type: "choice", instructions, criteria: { <label>: <meaning> } } | { type: "score", instructions, criteria: [<lowest level>, ..., <highest level>] } | { type: "bool", instructions, criteria: { true: <meaning>, false: <meaning> } } } })` resolves to `{ answers: { <id>: ... }, stopReason, errorMessage?, usage? }`.',
+	'- `models.generateImages(model, { input: [{ type: "text", text: <prompt> }, ...optional { type: "image", data, mimeType }] })` resolves to `{ output: [image blocks], stopReason, errorMessage?, usage? }`; show each image with `image(block)`.',
+	`At most ${MAX_CONCURRENT_MODEL_CALLS} classify/generateImages calls run at once; more wait.`,
+].join("\n");
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/** A script value in an error message: `undefined`, `a string`, `an array`, or its keys (`{ prompt }`). */
+function describeValue(value: unknown): string {
+	if (value === undefined || value === null) return String(value);
+	if (Array.isArray(value)) return value.length === 0 ? "an empty array" : "an array";
+	if (typeof value === "object") {
+		const keys = Object.keys(value);
+		if (keys.length === 0) return "{}";
+		return `{ ${keys.slice(0, 6).join(", ")}${keys.length > 6 ? ", ..." : ""} }`;
+	}
+	return typeof value === "string" ? "a string" : `a ${typeof value}`;
+}
+
+/** `an image`, `a classifier`. */
+function withArticle(word: string): string {
+	return `${/^[aeiou]/.test(word) ? "an" : "a"} ${word}`;
+}
+
+function toModelType(value: unknown): ModelType {
+	if (typeof value === "string" && MODEL_TYPES.has(value)) return value as ModelType;
+	throw new Error(`Unknown model type ${JSON.stringify(value)}. Use "chat", "image", or "classifier".`);
+}
+
+function toProvider(value: unknown): string | undefined {
+	if (value === undefined || value === null) return undefined;
+	if (typeof value !== "string") throw new Error("provider must be a string");
+	return value;
+}
+
+/** Catalog entry for scripts. `headers` is dropped because configured headers can carry credentials. */
+function toModelInfo(model: AnyModel): JsonValue {
+	const info: Record<string, unknown> = { ...model };
+	delete info.headers;
+	return JSON.parse(JSON.stringify(info)) as JsonValue;
+}
+
+/** The catalog functions: what a script's call of `name` with `args` (the argument list) resolves to. */
+export async function catalogCall(
+	name: string,
+	args: unknown,
+	models: Models,
+	context: Context,
+): Promise<JsonValue | undefined> {
+	const list = Array.isArray(args) ? args : [];
+	const [type, provider, id] = list;
+	if (name === "models.getModelsOfType")
+		return models.getModelsOfType(toModelType(type), toProvider(provider)).map(toModelInfo);
+	if (name === "models.getAvailableOfType") {
+		const available = await models.getAvailableOfType(toModelType(type), toProvider(provider), {
+			...(context.abortSignal === undefined ? {} : { signal: context.abortSignal }),
+		});
+		return available.map(toModelInfo);
+	}
+	if (typeof provider !== "string" || typeof id !== "string") {
+		throw new Error(
+			`models.getModelOfType(type, provider, id) expects three strings, got (${list.map(describeValue).join(", ")}). The provider and the id are separate arguments, for example models.getModelOfType("classifier", "typesafe", "jev-latest").`,
+		);
+	}
+	const model = models.getModelOfType(toModelType(type), provider, id);
+	return model === undefined ? undefined : toModelInfo(model);
+}
+
+const CLASSIFIER_CONTEXT_SHAPE =
+	'{ state: { ... }, images?: [{ type: "image", data: <base64>, mimeType }], questions: { <id>: { type: "choice", instructions, criteria: { <label>: <meaning> } } | { type: "score", instructions, criteria: [<lowest level>, ..., <highest level>] } | { type: "bool", instructions, criteria: { true: <meaning>, false: <meaning> } } } }';
+
+/** Check a script's classifier context, so mistakes fail with the expected shape instead of a provider error. */
+function checkClassifierContext(context: unknown): ClassifierContext {
+	const fail = (problem: string) =>
+		new Error(`models.classify() ${problem}. Expected context: ${CLASSIFIER_CONTEXT_SHAPE}.`);
+	if (!isRecord(context)) throw fail(`expects a context object as its second argument, got ${describeValue(context)}`);
+	if (!isRecord(context.state)) throw fail(`context.state must be an object, got ${describeValue(context.state)}`);
+	const { images } = context;
+	if (images !== undefined) {
+		if (!Array.isArray(images)) throw fail(`context.images must be an array, got ${describeValue(images)}`);
+		images.forEach((image: unknown, index) => {
+			if (
+				!isRecord(image) ||
+				image.type !== "image" ||
+				typeof image.data !== "string" ||
+				typeof image.mimeType !== "string"
+			) {
+				throw fail(`context.images[${index}] must be an image block, got ${describeValue(image)}`);
+			}
+		});
+	}
+	const { questions } = context;
+	if (!isRecord(questions) || Object.keys(questions).length === 0) {
+		throw fail(`context.questions must map question IDs to questions, got ${describeValue(questions)}`);
+	}
+	const isStrings = (values: unknown[]) => values.length > 0 && values.every((value) => typeof value === "string");
+	for (const [id, question] of Object.entries(questions)) {
+		const at = `context.questions.${id}`;
+		if (!isRecord(question)) throw fail(`${at} must be a question object, got ${describeValue(question)}`);
+		if (typeof question.instructions !== "string") throw fail(`${at}.instructions must be a string`);
+		const { criteria } = question;
+		if (question.type === "choice") {
+			if (!isRecord(criteria) || !isStrings(Object.values(criteria))) {
+				throw fail(`${at} is a "choice" question, so criteria must map each label to its meaning`);
+			}
+		} else if (question.type === "score") {
+			if (!Array.isArray(criteria) || !isStrings(criteria)) {
+				throw fail(`${at} is a "score" question, so criteria must list the levels as strings, lowest first`);
+			}
+		} else if (question.type === "bool") {
+			if (!isRecord(criteria) || typeof criteria.true !== "string" || typeof criteria.false !== "string") {
+				throw fail(`${at} is a "bool" question, so criteria must be { true: string, false: string }`);
+			}
+		} else {
+			throw fail(`${at}.type must be "choice", "score", or "bool", got ${JSON.stringify(question.type)}`);
+		}
+	}
+	return context as unknown as ClassifierContext;
+}
+
+/** Check a script's image context, so mistakes such as `{ prompt }` fail with the expected shape. */
+function checkImagesContext(context: unknown): ImagesContext {
+	const fail = (problem: string) =>
+		new Error(
+			`models.generateImages() ${problem}. Expected context: { input: [{ type: "text", text: <prompt> }, ...optional { type: "image", data: <base64>, mimeType } references] }.`,
+		);
+	if (!isRecord(context)) throw fail(`expects a context object as its second argument, got ${describeValue(context)}`);
+	const { input } = context;
+	if (!Array.isArray(input) || input.length === 0) {
+		throw fail(`context.input must be a non-empty array of blocks, got ${describeValue(input)}`);
+	}
+	input.forEach((block: unknown, index) => {
+		if (isRecord(block) && block.type === "text" && typeof block.text === "string") return;
+		if (
+			isRecord(block) &&
+			block.type === "image" &&
+			typeof block.data === "string" &&
+			typeof block.mimeType === "string"
+		)
+			return;
+		throw fail(`context.input[${index}] must be a text or image block, got ${describeValue(block)}`);
+	});
+	return context as unknown as ImagesContext;
+}
+
+/**
+ * Resolve the script's model by provider and id only, so a script-supplied baseUrl or headers never receive the
+ * credentials, with errors that say what to do.
+ */
+function resolveModel<TType extends "classifier" | "image">(
+	name: string,
+	type: TType,
+	model: unknown,
+	models: Models,
+): ModelTypeMap[TType] {
+	const listHint = `List the ${type} models you can use with models.getAvailableOfType("${type}").`;
+	if (!isRecord(model) || typeof model.provider !== "string" || typeof model.id !== "string") {
+		const undefinedHint =
+			model === undefined || model === null
+				? " models.getModelOfType() returns undefined for an unknown provider or id."
+				: "";
+		throw new Error(
+			`${name}() expects ${withArticle(type)} model as its first argument, got ${describeValue(model)}.${undefinedHint} ${listHint}`,
+		);
+	}
+	const { provider, id } = model;
+	const resolved = models.getModelOfType(type, provider, id);
+	if (resolved !== undefined) return resolved;
+	const actualType = [...MODEL_TYPES].find(
+		(other) => other !== type && models.getModelOfType(other as ModelType, provider, id) !== undefined,
+	);
+	throw new Error(
+		actualType
+			? `"${provider}/${id}" is ${withArticle(actualType)} model, not ${withArticle(type)} model. ${listHint}`
+			: `Unknown ${type} model "${provider}/${id}". ${listHint}`,
+	);
+}
+
+const ModelCallParameters = Type.Object({ model: Type.Unknown(), context: Type.Unknown() });
+
+/** The result of a model call as the tool returns it: the whole result for the script, usage counted by the Harness. */
+function modelResult(
+	label: string,
+	result: { stopReason: string; errorMessage?: string; usage?: Usage },
+	value: unknown,
+) {
+	const ok = result.stopReason === "stop";
+	return {
+		output: [
+			{
+				type: "text" as const,
+				text: ok ? `${label} done` : `${label} ${result.stopReason}: ${result.errorMessage ?? ""}`,
+			},
+		],
+		structuredOutput: JSON.parse(JSON.stringify(value)) as JsonValue,
+		...(ok ? {} : { isError: true }),
+		...(result.usage === undefined ? {} : { usage: result.usage }),
+	};
+}
+
+/** The internal tools behind `models.classify()` and `models.generateImages()`. */
+export function createModelTools(): ToolRegistration[] {
+	const classify = defineTool({
+		name: CLASSIFY_TOOL,
+		description: "models.classify() of code mode scripts.",
+		parameters: ModelCallParameters,
+		structuredOutputSchema: Type.Unknown(),
+		callers: ["tools"],
+		execute: async (args, api, context) => {
+			const model = resolveModel("models.classify", "classifier", args.model, api.models);
+			const checked = checkClassifierContext(args.context);
+			const result = await api.models.classify(model, checked, {
+				...(context.abortSignal === undefined ? {} : { signal: context.abortSignal }),
+			});
+			return modelResult(`classify ${model.provider}/${model.id}`, result, result);
+		},
+	});
+	const generateImages = defineTool({
+		name: GENERATE_IMAGES_TOOL,
+		description: "models.generateImages() of code mode scripts.",
+		parameters: ModelCallParameters,
+		structuredOutputSchema: Type.Unknown(),
+		callers: ["tools"],
+		execute: async (args, api, context) => {
+			const model = resolveModel("models.generateImages", "image", args.model, api.models);
+			const checked = checkImagesContext(args.context);
+			const result = await api.models.generateImages(model, checked, {
+				...(context.abortSignal === undefined ? {} : { signal: context.abortSignal }),
+			});
+			return modelResult(`generateImages ${model.provider}/${model.id}`, result, result);
+		},
+	});
+	return [classify as unknown as ToolRegistration, generateImages as unknown as ToolRegistration];
+}

third_party/pi/packages/durable/src/codemode/output.ts

@@ -0,0 +1,206 @@
+/**
+ * A script's result as the model reads it, copied from the coding agent's code mode: a "Script completed/failed"
+ * header with the wall time, the script's text items (`==> text N/M <==` when there are several), its `console` lines
+ * in one `<console_output>` block, the returned value, and on failure the error with the calls already made. Text past
+ * the token budget is cut to its start and end, with the complete text in a temp file; images are saved to temp files
+ * too, with their path next to them, because a script has no other way to hand their bytes on.
+ */
+import type { ImageContent, TextContent } from "@earendil-works/pi-ai";
+import type { CodemodeOutputItem, CodemodeResult } from "@earendil-works/pi-codemode/portable";
+
+type Item = TextContent | ImageContent;
+
+/** Writes a file of the result under a fixed name; its path, or why it could not be written. */
+export type OutputFile = (name: string, data: string | Uint8Array) => Promise<{ path: string } | { error: string }>;
+
+/** The tool's output limits, which the Harness enforces by cutting without a spill file: results stay inside them. */
+type Limits = { readonly maxBytes: number; readonly maxLines: number };
+/** Lines of the result besides the script's text: the header, markers, notes. */
+const LINE_MARGIN = 100;
+
+/** Default token budget for script output. */
+export const DEFAULT_MAX_OUTPUT_TOKENS = 10_000;
+/** Characters per token when estimating. */
+const CHARS_PER_TOKEN = 4;
+
+/** File extensions of the image types `image()` accepts. */
+const IMAGE_EXTENSIONS: Record<string, string> = {
+	"image/png": ".png",
+	"image/jpeg": ".jpg",
+	"image/gif": ".gif",
+	"image/webp": ".webp",
+};
+
+/** Like the script's `text()`: strings as is, other values as compact JSON. */
+function valueText(value: unknown): string {
+	if (typeof value === "string") return value;
+	return JSON.stringify(value) ?? String(value);
+}
+
+/**
+ * Lay out the script's output so the model can tell items apart: with more than one text item, each starts with a
+ * `==> text N/M <==` line; `console` lines follow all other output in one `<console_output>` block.
+ */
+function formatItems(output: readonly CodemodeOutputItem[]): Item[] {
+	const total = output.filter((item) => item.type === "text" && !item.console).length;
+	const items: Item[] = [];
+	const consoleLines: string[] = [];
+	let index = 0;
+	for (const item of output) {
+		if (item.type === "image") items.push(item);
+		else if (item.console) consoleLines.push(item.text);
+		else {
+			index++;
+			items.push({ type: "text", text: total > 1 ? `==> text ${index}/${total} <==\n${item.text}` : item.text });
+		}
+	}
+	if (consoleLines.length > 0) {
+		items.push({ type: "text", text: `<console_output>\n${consoleLines.join("\n")}\n</console_output>` });
+	}
+	return items;
+}
+
+/**
+ * Join adjacent text items into one, each part starting on its own line. The separator check looks at the last
+ * non-empty part, not the joined text: reading the end of a growing concatenation makes V8 copy it on every item, which
+ * is quadratic.
+ */
+function joinAdjacentText(items: readonly Item[]): Item[] {
+	const joined: Item[] = [];
+	let parts: string[] = [];
+	let lastPart = "";
+	const flush = () => {
+		if (parts.length > 0) joined.push({ type: "text", text: parts.join("") });
+		parts = [];
+		lastPart = "";
+	};
+	for (const item of items) {
+		if (item.type !== "text") {
+			flush();
+			joined.push(item);
+			continue;
+		}
+		if (lastPart !== "" && !lastPart.endsWith("\n")) {
+			parts.push("\n");
+			lastPart = "\n";
+		}
+		parts.push(item.text);
+		if (item.text !== "") lastPart = item.text;
+	}
+	flush();
+	return joined;
+}
+
+/**
+ * Apply the token budget: when the combined text exceeds it, the text items become one item that keeps the start and
+ * end of the text, and images follow it. The complete text is written to a temp file.
+ */
+async function truncate(
+	items: readonly Item[],
+	maxTokens: number,
+	limits: Limits,
+	write: OutputFile,
+): Promise<{ items: Item[]; fullOutputPath?: string }> {
+	const texts = items.filter((item): item is TextContent => item.type === "text").map((item) => item.text);
+	const combined = texts.join("\n");
+	// Within the limits too: a UTF-16 unit is at most 3 UTF-8 bytes, and the rest of the result is a few short lines.
+	const budget = Math.min(maxTokens * CHARS_PER_TOKEN, Math.floor(limits.maxBytes / 4));
+	const lineBudget = Math.floor((limits.maxLines - LINE_MARGIN) / 2);
+	const lines = combined.split("\n").length;
+	if (texts.length === 0 || (combined.length <= budget && lines <= 2 * lineBudget)) return { items: [...items] };
+	const headChars = Math.floor(budget / 2);
+	const tailChars = budget - headChars;
+	const head = combined.slice(0, headChars).split("\n").slice(0, lineBudget).join("\n");
+	const tail = tailChars > 0 ? combined.slice(-tailChars).split("\n").slice(-lineBudget).join("\n") : "";
+	const removed = combined.length - head.length - tail.length;
+	let text = `Warning: truncated output (original token count: ${Math.ceil(combined.length / CHARS_PER_TOKEN)})\nTotal output lines: ${combined.split("\n").length}\n\n${head}…${Math.ceil(removed / CHARS_PER_TOKEN)} tokens truncated…${tail}`;
+	const spilled = await write("output.txt", combined);
+	text +=
+		"path" in spilled
+			? `\n\n[Full output: ${spilled.path} (read with offset/limit)]`
+			: `\n\n[Could not save the full output: ${spilled.error}]`;
+	return {
+		items: [{ type: "text", text }, ...items.filter((item) => item.type === "image")],
+		...("path" in spilled ? { fullOutputPath: spilled.path } : {}),
+	};
+}
+
+/** Save each image to a file and put a text item with its path before it; an image shown twice is saved once. */
+async function saveImages(items: readonly Item[], write: OutputFile): Promise<Item[]> {
+	const labels = new Map<string, Promise<string>>();
+	const label = async ({ data, mimeType }: ImageContent, index: number): Promise<string> => {
+		const bytes = Uint8Array.from(atob(data), (char) => char.charCodeAt(0));
+		const kind = `${mimeType}, ${bytes.length} bytes`;
+		// A failed write must not discard the result of a script whose calls already ran: it becomes part of the label.
+		const saved = await write(`image-${index}${IMAGE_EXTENSIONS[mimeType] ?? ".bin"}`, bytes);
+		return "path" in saved
+			? `[Image saved to ${saved.path} (${kind})]`
+			: `[Image (${kind}) could not be saved: ${saved.error}]`;
+	};
+	const result = await Promise.all(
+		items.map(async (item): Promise<Item[]> => {
+			if (item.type !== "image") return [item];
+			let pending = labels.get(item.data);
+			if (pending === undefined) {
+				pending = label(item, labels.size + 1);
+				labels.set(item.data, pending);
+			}
+			return [{ type: "text", text: await pending }, item];
+		}),
+	);
+	return result.flat();
+}
+
+function formatError(
+	result: Extract<CodemodeResult, { ok: false }>,
+	calls: readonly { name: string; status: string }[],
+) {
+	const { error } = result;
+	const head =
+		error.kind === "script"
+			? (error.stack ?? `${error.name ?? "Error"}: ${error.message}`)
+			: error.kind === "timeout"
+				? `Script timed out: ${error.message}`
+				: error.kind === "aborted"
+					? `Script aborted: ${error.message}`
+					: `Script sandbox failed: ${error.message}`;
+	const summary =
+		calls.length === 0
+			? "No tool calls were made."
+			: `Tool calls made before the failure (they are not undone): ${calls.map((call) => `${call.name} (${call.status})`).join(", ")}`;
+	return `${head}\n\n${summary}`;
+}
+
+/** The tool result's content for a script's result, and the path of the complete text when it was cut. */
+export async function formatResult(
+	result: CodemodeResult,
+	options: {
+		/** Images `models.generateImages()` returned; when the script showed none, the result says so. */
+		readonly generatedImages: number;
+		readonly wallMs: number;
+		readonly maxOutputTokens: number;
+		readonly calls: readonly { name: string; status: string }[];
+		readonly limits: Limits;
+		readonly write: OutputFile;
+	},
+): Promise<{ content: Item[]; fullOutputPath?: string }> {
+	const output = [...result.output];
+	if (result.ok && result.value !== undefined) output.push({ type: "text", text: valueText(result.value) });
+	const items = formatItems(output);
+	if (!result.ok) items.push({ type: "text", text: `Script error:\n${formatError(result, options.calls)}` });
+	const generated = options.generatedImages;
+	if (generated > 0 && !items.some((item) => item.type === "image")) {
+		items.push({
+			type: "text",
+			text: `Note: models.generateImages() returned ${generated} image${generated === 1 ? "" : "s"} that the script did not show. Show each image block of result.output with image(block).`,
+		});
+	}
+	const truncated = await truncate(joinAdjacentText(items), options.maxOutputTokens, options.limits, options.write);
+	// After truncation, which joins the text items and moves images after them, so each path stays next to its image.
+	const content = joinAdjacentText(await saveImages(truncated.items, options.write));
+	const header = `${result.ok ? "Script completed" : "Script failed"}\nWall time ${(options.wallMs / 1000).toFixed(1)} seconds\nOutput:\n`;
+	return {
+		content: [{ type: "text", text: header }, ...content],
+		...(truncated.fullOutputPath === undefined ? {} : { fullOutputPath: truncated.fullOutputPath }),
+	};
+}

third_party/pi/packages/durable/src/documents.ts

@@ -0,0 +1,208 @@
+import { copyJson, type JsonValue } from "@earendil-works/chord";
+import type { Op } from "@earendil-works/chord/delta";
+import { idFromNumber } from "./ids.ts";
+import type {
+	CheckpointInfo,
+	CommonDocDefinition,
+	ConversationDocFamilyToken,
+	ConversationDocToken,
+	ConversationId,
+	DocDefinition,
+	DocFamilyDefinition,
+	DocFamilyToken,
+	DocToken,
+	DocumentAddress,
+	DocumentCreate,
+	DocumentId,
+	DocumentRecord,
+	DocumentSemantics,
+	Id,
+	JsonObject,
+	LatestConversationSemantics,
+	RewindableConversationDocFamilyToken,
+	RewindableConversationDocToken,
+	RewindableConversationSemantics,
+	SessionDocFamilyToken,
+	SessionDocToken,
+	StoredDocument,
+	TaskDocFamilyToken,
+	TaskDocToken,
+	TaskId,
+} from "./types.ts";
+
+type FamilyInput<T extends JsonObject, I extends JsonValue> = Omit<CommonDocDefinition<T>, "initial"> & {
+	readonly family: true;
+	initial(seed: I): T;
+};
+
+/** Define a Session-scoped singleton document. */
+export function defineDoc<T extends JsonObject>(
+	definition: CommonDocDefinition<T> & { readonly scope: "session" },
+): SessionDocToken<T>;
+/** Define a latest-only conversation singleton document. */
+export function defineDoc<T extends JsonObject>(
+	definition: CommonDocDefinition<T> & LatestConversationSemantics,
+): ConversationDocToken<T>;
+/** Define a rewindable conversation singleton document. */
+export function defineDoc<T extends JsonObject>(
+	definition: CommonDocDefinition<T> & RewindableConversationSemantics,
+): RewindableConversationDocToken<T>;
+/** Define a task-scoped singleton document. */
+export function defineDoc<T extends JsonObject>(
+	definition: CommonDocDefinition<T> & { readonly scope: "task" },
+): TaskDocToken<T>;
+export function defineDoc<T extends JsonObject>(definition: DocDefinition<T>): DocToken<T, DocDefinition<T>> {
+	validateDefinition(definition);
+	return { definition };
+}
+
+/** Define a Session-scoped document family. */
+export function defineDocFamily<T extends JsonObject, I extends JsonValue>(
+	definition: FamilyInput<T, I> & { readonly scope: "session" },
+): SessionDocFamilyToken<T, I>;
+/** Define a latest-only conversation document family. */
+export function defineDocFamily<T extends JsonObject, I extends JsonValue>(
+	definition: FamilyInput<T, I> & LatestConversationSemantics,
+): ConversationDocFamilyToken<T, I>;
+/** Define a rewindable conversation document family. */
+export function defineDocFamily<T extends JsonObject, I extends JsonValue>(
+	definition: FamilyInput<T, I> & RewindableConversationSemantics,
+): RewindableConversationDocFamilyToken<T, I>;
+/** Define a task-scoped document family. */
+export function defineDocFamily<T extends JsonObject, I extends JsonValue>(
+	definition: FamilyInput<T, I> & { readonly scope: "task" },
+): TaskDocFamilyToken<T, I>;
+export function defineDocFamily<T extends JsonObject, I extends JsonValue>(
+	definition: DocFamilyDefinition<T, I>,
+): DocFamilyToken<T, I, DocFamilyDefinition<T, I>> {
+	validateDefinition(definition);
+	return { definition };
+}
+
+/** Erased definition shape used by the Session after overload resolution. */
+export type AnyDocDefinition = DocumentSemantics & {
+	readonly kind: string;
+	readonly version: number;
+	readonly family?: true;
+	initial(seed?: JsonValue): JsonObject;
+	migrate?(value: JsonObject, fromVersion: number): JsonObject;
+	checkpointWhen?(value: Readonly<JsonObject>, ops: readonly Op[], info: CheckpointInfo): boolean;
+};
+
+/** Erased singleton or family token. */
+export type AnyDocToken = { readonly definition: AnyDocDefinition };
+
+function validateDefinition(definition: AnyDocDefinition): void {
+	if (!Number.isSafeInteger(definition.version) || definition.version < 1) {
+		throw new TypeError(`Document ${definition.kind} version must be a positive integer`);
+	}
+}
+
+/** Logical address plus its string identity for maps. */
+export type ResolvedAddress = {
+	readonly address: DocumentAddress;
+	readonly id: string;
+	readonly nextArgument: number;
+};
+
+/** Resolve an overloaded argument list and return the index after the owner and family key. */
+export function resolveAddress(definition: AnyDocDefinition, args: readonly unknown[]): ResolvedAddress {
+	let index = 0;
+	let scope: DocumentAddress["scope"];
+	switch (definition.scope) {
+		case "session":
+			scope = { kind: "session" };
+			break;
+		case "conversation":
+			scope = { kind: "conversation", conversationId: ownerId<ConversationId>(args[index++], definition) };
+			break;
+		case "task":
+			scope = { kind: "task", taskId: ownerId<TaskId>(args[index++], definition) };
+			break;
+	}
+	const key = definition.family === true ? (args[index++] as string) : undefined;
+	const address: DocumentAddress =
+		key === undefined ? { kind: definition.kind, scope } : { kind: definition.kind, scope, key };
+	return { address, id: addressId(address), nextArgument: index };
+}
+
+function ownerId<I extends Id<string>>(value: unknown, definition: AnyDocDefinition): I {
+	if (typeof value !== "number" || !Number.isSafeInteger(value)) {
+		throw new TypeError(`Document ${definition.kind} requires a ${definition.scope} ID`);
+	}
+	return idFromNumber<I>(value);
+}
+
+/** Stable string identity of one logical address. */
+export function addressId(address: DocumentAddress): string {
+	const owner =
+		address.scope.kind === "session"
+			? null
+			: address.scope.kind === "conversation"
+				? address.scope.conversationId
+				: address.scope.taskId;
+	return JSON.stringify([address.kind, address.scope.kind, owner, address.key ?? null]);
+}
+
+/** Build the storage create record for a new incarnation at an address. */
+export function documentCreate(definition: AnyDocDefinition, address: DocumentAddress, id: DocumentId): DocumentCreate {
+	const key = address.key === undefined ? {} : { key: address.key };
+	switch (address.scope.kind) {
+		case "session":
+			return { id, kind: address.kind, ...key, scope: address.scope };
+		case "task":
+			return { id, kind: address.kind, ...key, scope: address.scope };
+		case "conversation":
+			return {
+				id,
+				kind: address.kind,
+				...key,
+				scope: address.scope,
+				history: definition.history!,
+				fork: definition.fork!,
+			} as DocumentCreate;
+	}
+}
+
+/** Reject typed access whose token disagrees with the persisted scope, history, or fork semantics. */
+export function checkRecordScope(definition: AnyDocDefinition, record: DocumentCreate | DocumentRecord): void {
+	if (
+		record.scope.kind !== definition.scope ||
+		(record.scope.kind === "conversation" &&
+			(record.history !== definition.history || record.fork !== definition.fork))
+	) {
+		throw new TypeError(`Document ${record.id} (${record.kind}) does not match the supplied definition semantics`);
+	}
+}
+
+/** Reject typed access to a stored version the supplied definition cannot use. */
+export function checkRecordVersion(
+	definition: AnyDocDefinition,
+	record: DocumentCreate | DocumentRecord,
+	version: number,
+): void {
+	if (version > definition.version) {
+		throw new Error(`Document ${record.id} (${record.kind}) has newer version ${version} than ${definition.version}`);
+	}
+	if (version < definition.version && definition.migrate === undefined) {
+		throw new Error(`Document ${record.id} (${record.kind}) requires migration from version ${version}`);
+	}
+}
+
+/** Validate and materialize a detached stored value for typed access. */
+export function materializeDocument(definition: AnyDocDefinition, stored: StoredDocument): JsonObject {
+	return materializeDocumentValue(definition, stored.record, stored.version, stored.value);
+}
+
+/** Validate and materialize one detached value before its first persisted incarnation. */
+export function materializeDocumentValue(
+	definition: AnyDocDefinition,
+	record: DocumentCreate | DocumentRecord,
+	version: number,
+	value: JsonObject,
+): JsonObject {
+	checkRecordScope(definition, record);
+	checkRecordVersion(definition, record, version);
+	if (version === definition.version) return value;
+	return copyJson(definition.migrate!(value, version)) as JsonObject;
+}

third_party/pi/packages/durable/src/entries.ts

@@ -0,0 +1,34 @@
+import type { JsonValue } from "@earendil-works/chord";
+import type { CompactionReason, ToolDiagnostic } from "./harness/types.ts";
+import type { Entry, EntryRecord, TypedEntry } from "./types.ts";
+
+/** Define a typed entry kind whose `is()` guard narrows by `EntryRecord.kind`. */
+export function defineEntry<D extends JsonValue = never>(kind: string): Entry<D> {
+	if (typeof kind !== "string" || kind.length === 0) throw new TypeError("Entry kind must be a non-empty string");
+	return {
+		kind,
+		is: (entry: EntryRecord | undefined): entry is TypedEntry<D> => entry !== undefined && entry.kind === kind,
+	};
+}
+
+/** User input: `model` is `[UserMessage]`. Written by submissions. */
+export const UserEntry = defineEntry("pi.user");
+/** Provider result with any stop reason: `model` is `[AssistantMessage]`. Written by generation. */
+export const AssistantEntry = defineEntry("pi.assistant");
+/** Positional prompt and tool change: `model` is `[SystemMessage]` with empty `content`. */
+export const SystemEntry = defineEntry("pi.system");
+/**
+ * Tool result: `model` is `[ToolResultMessage]`, whose content ends with the rendered diagnostics block; `data` holds
+ * the structured diagnostics, possibly none. Written by tool tasks, and by generation for calls it did not offer.
+ */
+export const ToolResultEntry = defineEntry<{ diagnostics: ToolDiagnostic[] }>("pi.tool-result");
+/**
+ * Start of a new context: always `head: "self"`, with `model` absent for a plain reset or `[UserMessage]` carrying the
+ * handoff text. Written by `Conversation.reset()` and the `handoff` tool control.
+ */
+export const ResetEntry = defineEntry("pi.reset");
+/**
+ * Compaction summary: `model` is `[UserMessage]` with the wrapped summary, `head` the first kept entry. Written by
+ * compaction tasks, directly or through a write submission.
+ */
+export const CompactionEntry = defineEntry<{ reason: CompactionReason }>("pi.compaction");

third_party/pi/packages/durable/src/env/decode.ts

@@ -0,0 +1,30 @@
+/**
+ * Decoding that matches `new TextDecoder().decode(bytes)` of the whole input when the input arrives in pieces. Node's
+ * streaming decoder with BOM handling can drop a U+FEFF that follows a chunk boundary, not only a leading byte-order
+ * mark, so these decoders turn BOM handling off and drop a leading mark themselves.
+ */
+
+/** A streaming decoder for a byte range; callers that start at the beginning of a file skip a leading mark themselves. */
+export function rangeDecoder(): InstanceType<typeof TextDecoder> {
+	return new TextDecoder("utf-8", { ignoreBOM: true });
+}
+
+/** Whether decoding the whole input drops its first three bytes as a byte-order mark. */
+export function startsWithBom(firstBytes: Uint8Array): boolean {
+	return firstBytes[0] === 0xef && firstBytes[1] === 0xbb && firstBytes[2] === 0xbf;
+}
+
+/** Decodes one stream chunk by chunk exactly like decoding all of it at once. */
+export class StreamDecoder {
+	readonly #decoder = rangeDecoder();
+	#started = false;
+
+	/** Text for `bytes`, holding back an incomplete character; without `bytes`, the end of the stream. */
+	decode(bytes?: Uint8Array): string {
+		const text = bytes === undefined ? this.#decoder.decode() : this.#decoder.decode(bytes, { stream: true });
+		if (this.#started || text === "") return text;
+		this.#started = true;
+		// U+FEFF encodes only as EF BB BF, so a leading U+FEFF is exactly a byte-order mark at the stream's start.
+		return text.startsWith("\ufeff") ? text.slice(1) : text;
+	}
+}

third_party/pi/packages/durable/src/env/index.ts

@@ -0,0 +1,377 @@
+import type { Context } from "@earendil-works/chord";
+
+/** Result of a fallible operation. Expected failures are returned instead of thrown. */
+export type Result<TValue, TError> = { ok: true; value: TValue } | { ok: false; error: TError };
+
+export function ok<TValue, TError>(value: TValue): Result<TValue, TError> {
+	return { ok: true, value };
+}
+
+export function err<TValue, TError>(error: TError): Result<TValue, TError> {
+	return { ok: false, error };
+}
+
+export function getOrThrow<TValue, TError>(result: Result<TValue, TError>): TValue {
+	if (!result.ok) throw result.error;
+	return result.value;
+}
+
+export function getOrUndefined<TValue extends object, TError>(result: Result<TValue, TError>): TValue | undefined {
+	return result.ok ? result.value : undefined;
+}
+
+export function toError(error: unknown): Error {
+	if (error instanceof Error) return error;
+	if (typeof error === "string") return new Error(error);
+	try {
+		return new Error(JSON.stringify(error));
+	} catch {
+		return new Error(String(error));
+	}
+}
+
+export type FileKind = "file" | "directory" | "symlink";
+
+export type FileErrorCode =
+	| "aborted"
+	| "not_found"
+	| "permission_denied"
+	| "not_directory"
+	| "is_directory"
+	| "invalid"
+	| "not_supported"
+	| "unknown";
+
+export class FileError extends Error {
+	public code: FileErrorCode;
+	public path?: string;
+
+	constructor(code: FileErrorCode, message: string, path?: string, cause?: Error) {
+		super(message, cause === undefined ? undefined : { cause });
+		this.name = "FileError";
+		this.code = code;
+		this.path = path;
+	}
+}
+
+export type ExecutionErrorCode =
+	| "aborted"
+	| "timeout"
+	| "shell_unavailable"
+	| "spawn_error"
+	| "callback_error"
+	| "unknown";
+
+export class ExecutionError extends Error {
+	public code: ExecutionErrorCode;
+	/** Spill file of a command that timed out or was aborted after its output crossed the spill thresholds. */
+	public spillPath?: string;
+
+	constructor(code: ExecutionErrorCode, message: string, cause?: Error) {
+		super(message, cause === undefined ? undefined : { cause });
+		this.name = "ExecutionError";
+		this.code = code;
+	}
+}
+
+export interface FileInfo {
+	name: string;
+	path: string;
+	kind: FileKind;
+	size: number;
+	mtimeMs: number;
+}
+
+export interface TextLine {
+	text: string;
+	terminated: boolean;
+}
+
+export interface TextLineReader {
+	readLine(context: Context): Promise<Result<TextLine | undefined, FileError>>;
+	close(context: Context): Promise<void>;
+}
+
+/** Positional reads from one opened regular file; all calls see the same file even if its path is renamed. */
+export interface BinaryReader {
+	/** Metadata of the opened file, not of whatever its path names now. */
+	info(context: Context): Promise<Result<FileInfo, FileError>>;
+	/** Up to `length` bytes at `offset`; fewer only at end of file. */
+	read(offset: number, length: number, context: Context): Promise<Result<Uint8Array, FileError>>;
+	/**
+	 * One pass over the file that locates lines `[startLine, endLine)` (`endLine` absent: to the end), 0-based, where line
+	 * `k` starts after the `k`-th newline byte. Decoded sizes are those of the text `new TextDecoder().decode(file)` would
+	 * produce for that range, so a byte-order mark at the start of the file is not counted.
+	 */
+	scanLines(options: { startLine: number; endLine?: number }, context: Context): Promise<Result<LineScan, FileError>>;
+	close(context: Context): Promise<void>;
+}
+
+/** A file or directory to watch. It may be missing; creating it is a change. */
+export interface WatchTarget {
+	path: string;
+	/** Watch everything below a directory, not only its entries. Symbolic links below it are not followed. */
+	recursive?: boolean;
+	/** Entries below `path` that are neither watched nor reported: names starting with `.`, or these names. */
+	exclude?: { hidden?: boolean; names?: readonly string[] };
+}
+
+/**
+ * What changed:
+ * - `paths`: something at or below each path may have changed (a directory path covers its whole subtree). Calls may be
+ *   spurious; a change is never missed while the watcher is healthy.
+ * - `overflow`: coverage was uncertain for a while (lost events, reconnect); rescan everything that is watched.
+ * - `error`: the watcher stopped, for example because the watched tree grew past the environment's limit; no calls
+ *   follow.
+ */
+export type WatchChange = { paths: string[] } | { overflow: true } | { error: FileError };
+
+export interface FileWatcher {
+	/**
+	 * `native`: changes are reported within about two seconds. `polling`: the environment compares snapshots, because
+	 * the file system does not report changes reliably (network and FUSE file systems); a change undone between two
+	 * snapshots can be missed.
+	 */
+	readonly mode: "native" | "polling";
+	/** Stop watching; no `onChange` call starts after this resolves. Idempotent. */
+	close(context: Context): Promise<void>;
+}
+
+/** Where lines of a file are, as `BinaryReader.scanLines` found them. */
+export interface LineScan {
+	/** Newline bytes in the whole file; it has `newlines + 1` lines. */
+	newlines: number;
+	/**
+	 * Byte range of the selected lines: from the start of the first to the end of the last, without the newline that
+	 * ends it. A selection past the last line is empty at the end of the file.
+	 */
+	start: number;
+	end: number;
+	/** Where the first selected line ends: its newline, or the end of the file. */
+	firstLineEnd: number;
+	/** Where the last selected line starts. */
+	lastLineStart: number;
+	/** UTF-8 byte length of the decoded selection and of its first line. */
+	selectedBytes: number;
+	firstLineBytes: number;
+}
+
+/** Pages of one directory's entries. */
+export interface DirReader {
+	/**
+	 * Up to `maxEntries` entries in the order the file system returns them, continuing where the previous call stopped.
+	 * `done` marks the end; it may come with the last entries or with an empty page. An entry that disappears before its
+	 * metadata is read is skipped, as are entries of unsupported kinds. After a failed or aborted call, close the reader.
+	 */
+	next(maxEntries: number, context: Context): Promise<Result<{ entries: FileInfo[]; done: boolean }, FileError>>;
+	close(context: Context): Promise<void>;
+}
+
+/** What `FileSystem.listFiles` lists. */
+export interface ListFilesOptions {
+	/**
+	 * Include names starting with `.` (on Windows also files with the hidden attribute); default true. Entries named
+	 * `.git` are never listed.
+	 */
+	hidden?: boolean;
+	/**
+	 * Inside a git work tree, skip what git ignores: the `.gitignore` files of the directory, its subdirectories and its
+	 * parents, `.git/info/exclude`, and the global excludes file; default true. Unlike `git ls-files`, a tracked file that
+	 * matches an ignore rule is skipped too. Outside work trees, `.gitignore` files have no effect.
+	 */
+	gitignore?: boolean;
+	/** Names of entries to skip with everything below them, such as `node_modules`. */
+	exclude?: readonly string[];
+	/** Most entries to list; default 50 000. */
+	maxEntries?: number;
+	/** Most bytes of the listing: each entry's UTF-8 path plus two; default 4 MiB, at most 15.9 MiB. */
+	maxBytes?: number;
+	/** Time after which the listing stops; default 10 000 ms. */
+	timeoutMs?: number;
+}
+
+/** One entry of a `FileListing`. */
+export interface FileListEntry {
+	/** Relative to the listed directory, with `/` separators on every system. */
+	path: string;
+	/** A symbolic link is listed as such and not followed. */
+	kind: FileKind;
+}
+
+export interface FileListing {
+	/** Depth first, parents before their entries, siblings in byte order of their names. */
+	entries: FileListEntry[];
+	/** A bound in `ListFilesOptions` ended the listing early. */
+	truncated: boolean;
+}
+
+/** Portable filesystem capability. Operations return failures rather than throwing. */
+export interface FileSystem {
+	/**
+	 * The file namespace: equal ids see the same files at the same paths, whatever their `cwd`. Every local Node
+	 * environment shares one id; each container or remote host has its own.
+	 */
+	readonly id: string;
+	cwd: string;
+	absolutePath(path: string, context: Context): Promise<Result<string, FileError>>;
+	joinPath(parts: string[], context: Context): Promise<Result<string, FileError>>;
+	readTextFile(path: string, context: Context): Promise<Result<string, FileError>>;
+	openTextLineReader(path: string, context: Context): Promise<Result<TextLineReader, FileError>>;
+	readTextLines(
+		path: string,
+		options: { maxLines?: number } | undefined,
+		context: Context,
+	): Promise<Result<string[], FileError>>;
+	readBinaryFile(path: string, context: Context): Promise<Result<Uint8Array, FileError>>;
+	/**
+	 * Open a regular file for bounded positional reads. A directory fails with `is_directory`, other non-regular files
+	 * with `invalid`. With `noFollow`, a symbolic link as the final path component fails with `invalid` instead of being
+	 * followed; earlier components are still resolved.
+	 */
+	openBinaryReader(
+		path: string,
+		options: { noFollow?: boolean } | undefined,
+		context: Context,
+	): Promise<Result<BinaryReader, FileError>>;
+	writeFile(path: string, content: string | Uint8Array, context: Context): Promise<Result<void, FileError>>;
+	appendFile(path: string, content: string | Uint8Array, context: Context): Promise<Result<void, FileError>>;
+	/** Truncate or extend a file to exactly `size` bytes. */
+	truncateFile(path: string, size: number, context: Context): Promise<Result<void, FileError>>;
+	/** Flush file contents and metadata needed to retrieve them from an open file handle. */
+	flushFile(path: string, context: Context): Promise<Result<void, FileError>>;
+	renameFile(sourcePath: string, destinationPath: string, context: Context): Promise<Result<void, FileError>>;
+	fileInfo(path: string, context: Context): Promise<Result<FileInfo, FileError>>;
+	listDir(path: string, context: Context): Promise<Result<FileInfo[], FileError>>;
+	/**
+	 * Optional: the files, directories and symbolic links below a directory, at any depth, for indexes such as file
+	 * completion. Symbolic links are not followed; other kinds of files are not listed; entries that cannot be read are
+	 * skipped. See `ListFilesOptions` for hidden and ignored files and for the bounds. Fails with `not_supported` when
+	 * the environment cannot list right now (an older remote daemon), so callers fall back to their own walk.
+	 */
+	listFiles?(
+		path: string,
+		options: ListFilesOptions | undefined,
+		context: Context,
+	): Promise<Result<FileListing, FileError>>;
+	openDirReader(path: string, context: Context): Promise<Result<DirReader, FileError>>;
+	/**
+	 * Report changes to files and directories, for hosts that load resources from the environment. When the returned
+	 * watcher exists, coverage is established: a host that watches before it loads cannot miss a change made during the
+	 * load. See `WatchChange` for what is reported and `FileWatcher.mode` for how reliably.
+	 */
+	watch(
+		targets: readonly WatchTarget[],
+		onChange: (change: WatchChange) => void,
+		context: Context,
+	): Promise<Result<FileWatcher, FileError>>;
+	canonicalPath(path: string, context: Context): Promise<Result<string, FileError>>;
+	exists(path: string, context: Context): Promise<Result<boolean, FileError>>;
+	createDir(
+		path: string,
+		options: { recursive?: boolean } | undefined,
+		context: Context,
+	): Promise<Result<void, FileError>>;
+	remove(
+		path: string,
+		options: { recursive?: boolean; force?: boolean } | undefined,
+		context: Context,
+	): Promise<Result<void, FileError>>;
+	createTempDir(prefix: string | undefined, context: Context): Promise<Result<string, FileError>>;
+	createTempFile(
+		options: { prefix?: string; suffix?: string } | undefined,
+		context: Context,
+	): Promise<Result<string, FileError>>;
+	cleanup(context: Context): Promise<void>;
+}
+
+/** Spill the complete output to a temporary file once it exceeds either threshold. */
+export interface ShellSpillOptions {
+	afterBytes: number;
+	/** Complete or partial lines. */
+	afterLines: number;
+}
+
+export interface ShellExecResult {
+	exitCode: number;
+	/** Temporary file holding the complete raw output, when the spill thresholds were exceeded. */
+	spillPath?: string;
+}
+
+export interface ShellExecOptions {
+	cwd?: string;
+	env?: Record<string, string>;
+	inheritEnv?: boolean;
+	timeout?: number;
+	/**
+	 * Every decoded chunk of stdout and stderr as it arrives, in arrival order, with the stream it came from: raw,
+	 * unbounded, and unthrottled. Each stream is decoded separately, so a character split across chunks survives.
+	 */
+	onOutput?: (text: string, context: Context, info: ShellOutputInfo) => void;
+	spill?: ShellSpillOptions;
+	/**
+	 * The caller keeps only this tail of the output, so the environment may omit output outside it and report the omission
+	 * as `info.skipped`. Without it, every chunk is delivered.
+	 */
+	window?: ShellOutputWindow;
+}
+
+/**
+ * The tail of the combined output a caller keeps, and how often it samples it. An environment that transfers output
+ * over a slow link uses it to omit what the caller would drop anyway and to send no faster than the caller commits.
+ */
+export interface ShellOutputWindow {
+	/** UTF-8 bytes of decoded text kept at the end of the output. */
+	maxBytes: number;
+	/** Lines kept at the end of the output. */
+	maxLines: number;
+	/** Minimum pause between the caller's samples of the output. */
+	minIntervalMs: number;
+	/** Each sample also pauses the caller in proportion to its size at this rate. */
+	bytesPerSecond: number;
+}
+
+/**
+ * Output an environment omitted, measured on the decoded text `onOutput` would have received: every U+FFFD counts as
+ * three bytes, and no sanitizing is applied.
+ */
+/** The shape of `OutputSkip` from `@earendil-works/chord/output`, which `OutputBuffer.push()` accepts. */
+export interface ShellOutputSkip {
+	/** UTF-8 byte length of the omitted text. */
+	bytes: number;
+	/** Newlines (U+000A) in the omitted text. */
+	newlines: number;
+	/** Whether the omitted text ends with a newline. */
+	endsWithNewline: boolean;
+}
+
+export interface ShellOutputInfo {
+	stream: "stdout" | "stderr";
+	/**
+	 * Output omitted immediately before this chunk, only with `window`. The chunk then holds all output after the
+	 * omission up to its end, and that is more than the window by at least one byte or one line: more than
+	 * `window.maxBytes` bytes or more than `window.maxLines` newlines. So the omitted text can never be in the kept tail.
+	 * The omission and such a chunk may span both streams in arrival order; `stream` then names the chunk's last stream.
+	 * Callers that need the streams apart do not pass `window`.
+	 */
+	skipped?: ShellOutputSkip;
+}
+
+export interface Shell {
+	/**
+	 * Run a command. A string runs through the environment's shell. An array runs `command[0]` directly with the rest
+	 * as its arguments, without a shell, so they reach the program unparsed. Aborting the context or a timeout kills
+	 * only this command's processes.
+	 */
+	exec(
+		command: string | readonly string[],
+		options: ShellExecOptions | undefined,
+		context: Context,
+	): Promise<Result<ShellExecResult, ExecutionError>>;
+	/** Kill every command this environment still runs; for its owner's shutdown, never for one request. */
+	cleanup(context: Context): Promise<void>;
+}
+
+export interface ExecutionEnv extends FileSystem, Shell {}
+
+export { rangeDecoder, StreamDecoder, startsWithBom } from "./decode.ts";
+export { LineScanner } from "./line-scan.ts";

third_party/pi/packages/durable/src/env/line-scan.ts

@@ -0,0 +1,145 @@
+import { rangeDecoder, startsWithBom } from "./decode.ts";
+import type { LineScan } from "./index.ts";
+
+const NEWLINE = 0x0a;
+const encoder = new TextEncoder();
+
+/** UTF-8 byte length of decoded text. */
+function decodedBytes(text: string): number {
+	return text === "" ? 0 : encoder.encode(text).length;
+}
+
+/**
+ * Computes a `LineScan` from a file's bytes fed in order, so an environment can scan any size in bounded memory. Decoded
+ * sizes use the same streaming WHATWG decoder as decoding the whole file at once.
+ */
+export class LineScanner {
+	readonly #startLine: number;
+	readonly #endLine: number;
+	#position = 0;
+	#newlines = 0;
+	#lineStart = 0;
+	#start: number | undefined;
+	#end: number | undefined;
+	#firstLineEnd: number | undefined;
+	#lastLineStart: number | undefined;
+	#selectedBytes = 0;
+	#firstLineBytes = 0;
+	#selection: InstanceType<typeof TextDecoder> | undefined;
+	#firstLine: InstanceType<typeof TextDecoder> | undefined;
+	/** The first bytes, held until it is known whether they are a byte-order mark. */
+	#head: number[] | undefined = [];
+	#bom = false;
+
+	/** `startLine` and `endLine` must be non-negative integers with `endLine > startLine`; `endLine` absent: to the end. */
+	constructor(startLine: number, endLine = Number.POSITIVE_INFINITY) {
+		if (!Number.isSafeInteger(startLine) || startLine < 0 || !(endLine > startLine)) {
+			throw new RangeError("Invalid line range");
+		}
+		if (endLine !== Number.POSITIVE_INFINITY && !Number.isSafeInteger(endLine))
+			throw new RangeError("Invalid line range");
+		this.#startLine = startLine;
+		this.#endLine = endLine;
+		if (startLine === 0) this.#begin(0);
+	}
+
+	push(chunk: Uint8Array): void {
+		if (this.#head !== undefined) {
+			const take = Math.min(3 - this.#head.length, chunk.length);
+			this.#head.push(...chunk.subarray(0, take));
+			if (this.#head.length < 3) return;
+			this.#releaseHead();
+			chunk = chunk.subarray(take);
+		}
+		this.#process(chunk);
+	}
+
+	#releaseHead(): void {
+		const head = Uint8Array.from(this.#head ?? []);
+		this.#head = undefined;
+		this.#bom = startsWithBom(head);
+		this.#process(head);
+	}
+
+	#process(chunk: Uint8Array): void {
+		const base = this.#position;
+		let from = 0;
+		for (let index = chunk.indexOf(NEWLINE); index !== -1; index = chunk.indexOf(NEWLINE, index + 1)) {
+			// The newline ends line `this.#newlines`. It belongs to the selection between selected lines only.
+			this.#feed(chunk, base, from, index);
+			const line = this.#newlines;
+			const position = base + index;
+			if (line === this.#startLine) this.#endFirstLine(position);
+			if (line === this.#endLine - 1) this.#endSelection(position);
+			this.#feed(chunk, base, index, index + 1);
+			from = index + 1;
+			this.#newlines++;
+			this.#lineStart = position + 1;
+			if (this.#newlines === this.#startLine) this.#begin(this.#lineStart);
+			if (this.#newlines === this.#endLine - 1) this.#lastLineStart = this.#lineStart;
+		}
+		this.#feed(chunk, base, from, chunk.length);
+		this.#position += chunk.length;
+	}
+
+	finish(): LineScan {
+		if (this.#head !== undefined) this.#releaseHead();
+		const size = this.#position;
+		if (this.#start === undefined) {
+			return {
+				newlines: this.#newlines,
+				start: size,
+				end: size,
+				firstLineEnd: size,
+				lastLineStart: size,
+				selectedBytes: 0,
+				firstLineBytes: 0,
+			};
+		}
+		if (this.#firstLineEnd === undefined) this.#endFirstLine(size);
+		if (this.#end === undefined) this.#endSelection(size);
+		return {
+			newlines: this.#newlines,
+			start: this.#start,
+			end: this.#end ?? size,
+			firstLineEnd: this.#firstLineEnd ?? size,
+			// A selection that reaches past the last line ends with the last line.
+			lastLineStart: this.#lastLineStart ?? this.#lineStart,
+			selectedBytes: this.#selectedBytes,
+			firstLineBytes: this.#firstLineBytes,
+		};
+	}
+
+	#begin(start: number): void {
+		this.#start = start;
+		if (this.#startLine === this.#endLine - 1) this.#lastLineStart = start;
+		this.#selection = rangeDecoder();
+		this.#firstLine = rangeDecoder();
+	}
+
+	#endFirstLine(position: number): void {
+		this.#firstLineEnd = position;
+		if (this.#firstLine !== undefined) this.#firstLineBytes += decodedBytes(this.#firstLine.decode());
+		this.#firstLine = undefined;
+	}
+
+	#endSelection(position: number): void {
+		this.#end = position;
+		if (this.#selection !== undefined) this.#selectedBytes += decodedBytes(this.#selection.decode());
+		this.#selection = undefined;
+	}
+
+	/** Feed `chunk[from, to)`, which starts at file offset `base`, to the decoders of the ranges still open. */
+	#feed(chunk: Uint8Array, base: number, from: number, to: number): void {
+		// Decoding the whole file drops a leading byte-order mark.
+		if (this.#bom && base + from < 3) from = Math.min(to, 3 - base);
+		if (to <= from) return;
+		const bytes = chunk.subarray(from, to);
+		if (this.#selection !== undefined) {
+			this.#selectedBytes += decodedBytes(this.#selection.decode(bytes, { stream: true }));
+		}
+		if (this.#firstLine !== undefined) {
+			this.#firstLineBytes += decodedBytes(this.#firstLine.decode(bytes, { stream: true }));
+		}
+	}
+}

third_party/pi/packages/durable/src/env/node-watch.ts

@@ -0,0 +1,522 @@
+import { createHash } from "node:crypto";
+import { type FSWatcher, watch as fsWatch, type Stats } from "node:fs";
+import { lstat, readdir, readFile, stat, statfs } from "node:fs/promises";
+import { dirname, relative, sep } from "node:path";
+import type { Context } from "@earendil-works/chord";
+import { FileError, type FileWatcher, type WatchChange, type WatchTarget } from "./index.ts";
+
+/** How `NodeExecutionEnv` watches. */
+export interface NodeWatchOptions {
+	/**
+	 * Force a mode. By default `polling` is chosen on Windows, where native watchers keep directories open and so block
+	 * renaming their parents, and for file systems that do not report remote changes.
+	 */
+	mode?: "native" | "polling";
+	/** Interval between snapshots in `polling` mode; default 2000 ms. */
+	pollIntervalMs?: number;
+	/** Most directories one watcher covers; default 10,000. */
+	maxDirectories?: number;
+}
+
+type ResolvedTarget = {
+	readonly path: string;
+	readonly recursive: boolean;
+	readonly hidden: boolean;
+	readonly names: ReadonlySet<string>;
+};
+
+/** What a snapshot remembers of one path. Directories and ancestors are compared by identity only. */
+type Entry = {
+	readonly kind: "file" | "directory" | "symlink" | "other";
+	readonly dev: number;
+	readonly ino: number;
+	readonly size: number;
+	readonly mtimeMs: number;
+	readonly hash?: string;
+};
+
+type Snapshot = Map<string, Entry>;
+
+/** A native watcher and the identity of what it was installed on: a path replaced by another file needs a new one. */
+type Installed = { readonly watcher: FSWatcher; readonly dev: number; readonly ino: number };
+
+/** A scan: the snapshot, and targets that are symbolic links to files, whose files need their own watchers. */
+type Scan = { readonly snapshot: Snapshot; readonly linkedFiles: ReadonlySet<string> };
+
+const DEBOUNCE_MS = 50;
+/**
+ * On macOS, `fs.watch` returns before libuv's FSEvents stream is live, and changes in between are never reported
+ * (about 2% of changes made right after the call, none after 100 ms). A rescan this long after installing watchers
+ * catches them.
+ */
+const FSEVENTS_SETTLE_MS = 500;
+const DEFAULT_POLL_MS = 2000;
+const DEFAULT_MAX_DIRECTORIES = 10_000;
+/** In polling mode, recently modified small files are also compared by content: a second write within the file system's
+ * timestamp granularity can keep size and modification time. */
+const HASH_MAX_BYTES = 256 * 1024;
+const HASH_RECENT_MS = 5000;
+
+/** Linux `statfs` magic numbers of file systems that accept watches but do not report changes made elsewhere. */
+const UNRELIABLE_FILE_SYSTEMS = new Set([
+	0x6969, // NFS
+	0x517b, // SMB
+	0xff534d42, // CIFS
+	0xfe534d42, // SMB2
+	0x65735546, // FUSE (sshfs, Android shared storage)
+	0x01021997, // 9P (WSL2 Windows drives)
+	0x0bd00bd0, // Lustre
+	0x47504653, // GPFS
+	0x00c36400, // Ceph
+	0x5346414f, // OpenAFS
+	0x6b414653, // kAFS
+	0x5dca2df5, // sdcardfs
+]);
+
+class BudgetExceeded extends Error {}
+
+function isNodeError(error: unknown): error is NodeJS.ErrnoException {
+	return error instanceof Error && "code" in error;
+}
+
+function isDenied(error: unknown): boolean {
+	return isNodeError(error) && (error.code === "EACCES" || error.code === "EPERM");
+}
+
+function entryOf(stats: Stats, hash?: string): Entry {
+	const kind = stats.isFile()
+		? "file"
+		: stats.isDirectory()
+			? "directory"
+			: stats.isSymbolicLink()
+				? "symlink"
+				: "other";
+	return {
+		kind,
+		dev: stats.dev,
+		ino: stats.ino,
+		size: kind === "directory" ? 0 : stats.size,
+		mtimeMs: kind === "directory" ? 0 : stats.mtimeMs,
+		...(hash === undefined ? {} : { hash }),
+	};
+}
+
+function sameEntry(a: Entry, b: Entry): boolean {
+	return (
+		a.kind === b.kind &&
+		a.dev === b.dev &&
+		a.ino === b.ino &&
+		a.size === b.size &&
+		a.mtimeMs === b.mtimeMs &&
+		a.hash === b.hash
+	);
+}
+
+/** Ancestors of `path`, nearest first, up to the root. */
+function ancestorsOf(path: string): string[] {
+	const result: string[] = [];
+	for (let current = dirname(path); ; current = dirname(current)) {
+		result.push(current);
+		if (dirname(current) === current) return result;
+	}
+}
+
+/** Whether `path` is `ancestor` or below it. */
+function isWithin(path: string, ancestor: string): boolean {
+	return path === ancestor || path.startsWith(ancestor.endsWith(sep) ? ancestor : ancestor + sep);
+}
+
+function excluded(target: ResolvedTarget, name: string): boolean {
+	return (target.hidden && name.startsWith(".")) || target.names.has(name);
+}
+
+/**
+ * Watches by snapshots: native events only trigger a debounced rescan, and changes are the difference between
+ * snapshots plus the event paths. A replaced file, a renamed or recreated ancestor, or a directory created with its
+ * contents therefore never depends on which events an operating system sends. New directories get their watchers
+ * before they are scanned again, so nothing written into them before the watcher existed is missed.
+ */
+export class NodeFileWatcher implements FileWatcher {
+	readonly #targets: readonly ResolvedTarget[];
+	readonly #onChange: (change: WatchChange) => void;
+	readonly #pollIntervalMs: number;
+	readonly #maxDirectories: number;
+	readonly #watchers = new Map<string, Installed>();
+	readonly #events = new Set<string>();
+	#mode: "native" | "polling";
+	#snapshot: Snapshot = new Map();
+	#timer: ReturnType<typeof setTimeout> | undefined;
+	#settleTimer: ReturnType<typeof setTimeout> | undefined;
+	#running: Promise<void> | undefined;
+	#dirty = false;
+	#closed = false;
+
+	private constructor(
+		targets: readonly ResolvedTarget[],
+		onChange: (change: WatchChange) => void,
+		mode: "native" | "polling",
+		options: NodeWatchOptions,
+	) {
+		this.#targets = targets;
+		this.#onChange = onChange;
+		this.#mode = mode;
+		this.#pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_MS;
+		this.#maxDirectories = options.maxDirectories ?? DEFAULT_MAX_DIRECTORIES;
+	}
+
+	/** Establish coverage: watchers first, then the snapshot later changes are compared with. */
+	static async open(
+		targets: readonly WatchTarget[],
+		resolvePath: (path: string) => string,
+		onChange: (change: WatchChange) => void,
+		options: NodeWatchOptions,
+	): Promise<NodeFileWatcher> {
+		const resolved = targets.map((target) => ({
+			path: resolvePath(target.path),
+			recursive: target.recursive === true,
+			hidden: target.exclude?.hidden === true,
+			names: new Set(target.exclude?.names ?? []),
+		}));
+		// Windows refuses to rename a directory while another directory below it is open, and a native watcher keeps
+		// every watched directory open: watching would break renames of their parents, so Windows polls.
+		const mode =
+			options.mode ??
+			(process.platform === "win32" || (await anyUnreliable(resolved.map((target) => target.path)))
+				? "polling"
+				: "native");
+		const watcher = new NodeFileWatcher(resolved, onChange, mode, options);
+		try {
+			await watcher.#sync(false);
+		} catch (error) {
+			watcher.#stop();
+			if (error instanceof BudgetExceeded) throw new FileError("invalid", error.message);
+			throw error;
+		}
+		watcher.#schedulePoll();
+		return watcher;
+	}
+
+	get mode(): "native" | "polling" {
+		return this.#mode;
+	}
+
+	async close(_context: Context): Promise<void> {
+		if (this.#closed) return;
+		this.#stop();
+		await this.#running;
+	}
+
+	#stop(): void {
+		this.#closed = true;
+		clearTimeout(this.#timer);
+		this.#timer = undefined;
+		clearTimeout(this.#settleTimer);
+		this.#settleTimer = undefined;
+		for (const { watcher } of this.#watchers.values()) watcher.close();
+		this.#watchers.clear();
+	}
+
+	#deliver(change: WatchChange): void {
+		if (this.#closed) return;
+		try {
+			this.#onChange(change);
+		} catch {
+			// A throwing callback must not stop watching.
+		}
+	}
+
+	#schedulePoll(): void {
+		if (this.#closed || this.#mode !== "polling") return;
+		this.#timer = setTimeout(() => {
+			this.#timer = undefined;
+			void this.#flush().finally(() => this.#schedulePoll());
+		}, this.#pollIntervalMs);
+	}
+
+	#scheduleFlush(): void {
+		if (this.#closed || this.#timer !== undefined) return;
+		this.#timer = setTimeout(() => {
+			this.#timer = undefined;
+			void this.#flush();
+		}, DEBOUNCE_MS);
+	}
+
+	#scheduleSettle(): void {
+		if (this.#closed) return;
+		clearTimeout(this.#settleTimer);
+		this.#settleTimer = setTimeout(() => {
+			this.#settleTimer = undefined;
+			void this.#flush();
+		}, FSEVENTS_SETTLE_MS);
+	}
+
+	#flush(): Promise<void> {
+		if (this.#running !== undefined) {
+			this.#dirty = true;
+			return this.#running;
+		}
+		this.#running = (async () => {
+			try {
+				do {
+					this.#dirty = false;
+					const events = [...this.#events];
+					this.#events.clear();
+					const changed = await this.#sync(true);
+					for (const path of events) changed.add(path);
+					if (changed.size > 0) this.#deliver({ paths: [...changed].sort() });
+				} while (this.#dirty && !this.#closed);
+			} catch (error) {
+				const message = error instanceof Error ? error.message : String(error);
+				const fileError =
+					error instanceof FileError
+						? error
+						: new FileError(isDenied(error) ? "permission_denied" : "invalid", message);
+				this.#deliver({ error: fileError });
+				this.#stop();
+			} finally {
+				this.#running = undefined;
+			}
+		})();
+		return this.#running;
+	}
+
+	/** Rescan, report differences, and install watchers for new directories, rescanning until none are new. */
+	async #sync(report: boolean): Promise<Set<string>> {
+		const changed = new Set<string>();
+		for (let round = 0; round < 10 && !this.#closed; round++) {
+			const scan = await this.#scan();
+			// Closed during the scan: installing watchers now would leak them.
+			if (this.#closed) break;
+			const next = scan.snapshot;
+			if (report) for (const path of this.#diff(this.#snapshot, next)) changed.add(path);
+			this.#snapshot = next;
+			if (this.#mode === "polling" || !this.#reconcileWatchers(scan)) break;
+			if (process.platform === "darwin") this.#scheduleSettle();
+			// Something written into a new directory before its watcher existed shows up in the next round.
+			report = true;
+		}
+		return changed;
+	}
+
+	#diff(previous: Snapshot, next: Snapshot): string[] {
+		const changed: string[] = [];
+		for (const [path, entry] of next) {
+			const before = previous.get(path);
+			if (before === undefined || !sameEntry(before, entry)) changed.push(this.#reported(path));
+		}
+		for (const path of previous.keys()) if (!next.has(path)) changed.push(this.#reported(path));
+		return changed;
+	}
+
+	/** An ancestor that changed identity moved every target below it; report those targets. */
+	#reported(path: string): string {
+		if (this.#targets.some((target) => isWithin(path, target.path))) return path;
+		return this.#targets.find((target) => isWithin(target.path, path))?.path ?? path;
+	}
+
+	async #scan(): Promise<Scan> {
+		const snapshot: Snapshot = new Map();
+		const linkedFiles = new Set<string>();
+		// Directories are counted once, but traversed once per target: overlapping targets differ in recursion and
+		// exclusions, and an entry recorded for one target must still be descended into for another.
+		const counted = new Set<string>();
+		const traversed = new Set<string>();
+		/** Kinds of listed entries, not following links: a target that links to a directory is recorded by `stat`, but
+		 * symbolic links below a recursive target are not followed. */
+		const listed = new Map<string, Entry["kind"]>();
+		const countDirectory = (path: string): void => {
+			counted.add(path);
+			if (counted.size > this.#maxDirectories) {
+				throw new BudgetExceeded(`Watched paths exceed ${this.#maxDirectories} directories`);
+			}
+		};
+		const record = async (path: string, stats: Stats): Promise<void> => {
+			let hash: string | undefined;
+			if (
+				this.#mode === "polling" &&
+				stats.isFile() &&
+				stats.size <= HASH_MAX_BYTES &&
+				Date.now() - stats.mtimeMs < HASH_RECENT_MS
+			) {
+				hash = await readFile(path).then(
+					(content) => createHash("sha256").update(content).digest("hex"),
+					() => undefined,
+				);
+			}
+			snapshot.set(path, entryOf(stats, hash));
+		};
+		const scanDirectory = async (index: number, target: ResolvedTarget, directory: string): Promise<void> => {
+			const key = `${index}\0${directory}`;
+			if (traversed.has(key)) return;
+			traversed.add(key);
+			let names: string[];
+			try {
+				names = await readdir(directory);
+			} catch (error) {
+				// The watched directory itself must be readable; below it, unreadable directories are skipped.
+				if (directory === target.path && isDenied(error)) throw error;
+				if (isNodeError(error) && ["ENOENT", "EACCES", "EPERM", "ENOTDIR"].includes(error.code ?? "")) return;
+				throw error;
+			}
+			for (const name of names) {
+				if (excluded(target, name)) continue;
+				const path = `${directory.endsWith(sep) ? directory : directory + sep}${name}`;
+				let kind = listed.get(path);
+				if (kind === undefined) {
+					const stats = await lstat(path).catch(() => undefined);
+					if (stats === undefined) continue;
+					kind = entryOf(stats).kind;
+					listed.set(path, kind);
+					// A target's own entry (following links) wins over its listing by another target.
+					if (!snapshot.has(path)) await record(path, stats);
+				}
+				if (target.recursive && kind === "directory") {
+					countDirectory(path);
+					await scanDirectory(index, target, path);
+				}
+			}
+		};
+		for (const [index, target] of this.#targets.entries()) {
+			for (const ancestor of ancestorsOf(target.path)) {
+				if (snapshot.has(ancestor)) continue;
+				const stats = await lstat(ancestor).catch(() => undefined);
+				// Identity only: an ancestor's own timestamps change with every unrelated sibling.
+				if (stats !== undefined) snapshot.set(ancestor, { ...entryOf(stats), size: 0, mtimeMs: 0 });
+			}
+			// The target itself may be a symbolic link to what is watched; follow it. A missing target is watched for its
+			// creation; one that cannot be reached for lack of permission fails.
+			let stats: Stats;
+			try {
+				stats = await stat(target.path);
+			} catch (error) {
+				if (isDenied(error)) throw error;
+				continue;
+			}
+			await record(target.path, stats);
+			if (stats.isFile() && (await lstat(target.path).catch(() => undefined))?.isSymbolicLink()) {
+				linkedFiles.add(target.path);
+			}
+			if (stats.isDirectory()) {
+				countDirectory(target.path);
+				await scanDirectory(index, target, target.path);
+			}
+		}
+		return { snapshot, linkedFiles };
+	}
+
+	/**
+	 * Watch every existing ancestor of each target, each target directory, each target that is a symbolic link to a file
+	 * (changes to that file are not events of the link's directory), and on Linux each directory below a recursive target
+	 * (elsewhere one recursive watcher per target). Returns whether a watcher was added.
+	 */
+	#reconcileWatchers({ snapshot, linkedFiles }: Scan): boolean {
+		const wanted = new Map<string, boolean>();
+		for (const path of linkedFiles) wanted.set(path, false);
+		const perDirectory = process.platform === "linux" || process.platform === "android";
+		for (const target of this.#targets) {
+			for (const ancestor of ancestorsOf(target.path)) {
+				if (snapshot.get(ancestor)?.kind === "directory" && !wanted.has(ancestor)) wanted.set(ancestor, false);
+			}
+			if (snapshot.get(target.path)?.kind !== "directory") continue;
+			const recursiveRoot = target.recursive && !perDirectory;
+			wanted.set(target.path, recursiveRoot);
+			if (target.recursive && perDirectory) {
+				for (const [path, entry] of snapshot) {
+					if (entry.kind === "directory" && path !== target.path && isWithin(path, target.path)) {
+						wanted.set(path, false);
+					}
+				}
+			}
+		}
+		for (const [path, installed] of this.#watchers) {
+			const entry = snapshot.get(path);
+			// Gone, or replaced: a watcher follows the directory it was installed on, not the path.
+			if (!wanted.has(path) || entry === undefined || entry.dev !== installed.dev || entry.ino !== installed.ino) {
+				installed.watcher.close();
+				this.#watchers.delete(path);
+			}
+		}
+		let added = false;
+		for (const [path, recursive] of wanted) {
+			const entry = snapshot.get(path);
+			if (this.#watchers.has(path) || entry === undefined) continue;
+			const linked = linkedFiles.has(path);
+			try {
+				const watcher = fsWatch(path, { recursive, persistent: false }, (_event, filename) =>
+					linked
+						? this.#onLinkedFileEvent(path)
+						: this.#onEvent(path, filename === null ? undefined : String(filename)),
+				);
+				// A watched directory that disappears or fails: rescan, which also replaces the watcher.
+				watcher.on("error", () => {
+					watcher.close();
+					if (this.#watchers.get(path)?.watcher === watcher) this.#watchers.delete(path);
+					this.#scheduleFlush();
+				});
+				this.#watchers.set(path, { watcher, dev: entry.dev, ino: entry.ino });
+				added = true;
+			} catch (error) {
+				// Out of watches or unsupported: compare snapshots from now on, and say coverage was uncertain.
+				if (isNodeError(error) && error.code !== "ENOENT" && error.code !== "EACCES" && error.code !== "EPERM") {
+					this.#switchToPolling();
+					return false;
+				}
+			}
+		}
+		return added;
+	}
+
+	#switchToPolling(): void {
+		if (this.#mode === "polling") return;
+		this.#mode = "polling";
+		for (const { watcher } of this.#watchers.values()) watcher.close();
+		this.#watchers.clear();
+		this.#deliver({ overflow: true });
+		clearTimeout(this.#timer);
+		this.#timer = undefined;
+		this.#schedulePoll();
+	}
+
+	#onEvent(directory: string, filename: string | undefined): void {
+		if (this.#closed) return;
+		const path =
+			filename === undefined ? directory : `${directory.endsWith(sep) ? directory : directory + sep}${filename}`;
+		// Events about unrelated siblings of an ancestor, or about excluded entries, are ignored.
+		const relevant = this.#inScope(path);
+		if (relevant) this.#events.add(this.#reported(path));
+		if (relevant || filename === undefined) this.#scheduleFlush();
+	}
+
+	/** The file a target links to changed: report the target. */
+	#onLinkedFileEvent(target: string): void {
+		if (this.#closed) return;
+		this.#events.add(target);
+		this.#scheduleFlush();
+	}
+
+	#inScope(path: string): boolean {
+		for (const target of this.#targets) {
+			if (isWithin(target.path, path)) return true;
+			if (!isWithin(path, target.path) || path === target.path) continue;
+			const components = relative(target.path, path).split(sep);
+			if (!target.recursive && components.length > 1) continue;
+			if (components.some((name) => excluded(target, name))) continue;
+			return true;
+		}
+		return false;
+	}
+}
+
+/** Whether any path, or its nearest existing ancestor, is on a file system that does not report remote changes. */
+async function anyUnreliable(paths: readonly string[]): Promise<boolean> {
+	if (process.platform !== "linux" && process.platform !== "android") return false;
+	for (const path of paths) {
+		for (const candidate of [path, ...ancestorsOf(path)]) {
+			const info = await statfs(candidate).catch(() => undefined);
+			if (info === undefined) continue;
+			if (UNRELIABLE_FILE_SYSTEMS.has(info.type)) return true;
+			break;
+		}
+	}
+	return false;
+}

third_party/pi/packages/durable/src/errors.ts

@@ -0,0 +1,57 @@
+import type { ConversationId } from "./types.ts";
+
+/**
+ * The message of a thrown value, without throwing itself: `String()` throws for values such as `Object.create(null)`,
+ * and code that formats an error must not fail for the value it formats.
+ */
+export function errorMessage(error: unknown): string {
+	if (error instanceof Error) return error.message;
+	try {
+		return String(error);
+	} catch {
+		return Object.prototype.toString.call(error);
+	}
+}
+
+/** A transaction read a table after its first table write. Read every required row before writing. */
+export class ReadAfterWrite extends Error {
+	constructor(method: string) {
+		super(`Tx.${method}() cannot read tables after the first table write`);
+		this.name = "ReadAfterWrite";
+	}
+}
+
+/**
+ * A Storage read rejected an invalid request, such as an unknown conversation or a cursor from another scan, with no
+ * durable effect. Unlike any other error a Storage throws, it fails only that read, not the Session; from `commit()` it is
+ * fatal like any other.
+ */
+export class StorageRequestError extends Error {
+	constructor(message: string) {
+		super(message);
+		this.name = "StorageRequestError";
+	}
+}
+
+/**
+ * The Session failed, so nothing runs or commits any more. It fails when a Storage method throws, a commit cannot be
+ * adopted, or the Harness's own bookkeeping throws (a scheduler commit, an internal commit listener, a registry whose
+ * `snapshot()` throws). `cause` is that first error. Reopen the Session; reopening recovers from what was committed.
+ */
+export class SessionFailed extends Error {
+	constructor(cause: unknown) {
+		super("Session failed after a storage error; close and reopen it", { cause });
+		this.name = "SessionFailed";
+	}
+}
+
+/** A submission reached a busy conversation and was not admitted. */
+export class ConversationBusy extends Error {
+	readonly conversationId: ConversationId;
+
+	constructor(conversationId: ConversationId) {
+		super(`Conversation ${conversationId} is busy`);
+		this.name = "ConversationBusy";
+		this.conversationId = conversationId;
+	}
+}

third_party/pi/packages/durable/src/harness/agent.ts

@@ -0,0 +1,314 @@
+import { copyJson, type Draft } from "@earendil-works/chord";
+import { defineDoc } from "../documents.ts";
+import { documentRetention } from "../session/session.ts";
+import type { ConversationId, ConversationRecord, Tx } from "../types.ts";
+import type {
+	Agent,
+	AgentChange,
+	AgentState,
+	CompactionPolicy,
+	ConversationRetryPolicy,
+	Extension,
+	HarnessSettings,
+	ProgressPolicy,
+	PromptSection,
+	RegistrySnapshot,
+	Settings,
+	ToolRegistration,
+} from "./types.ts";
+
+export const DEFAULT_RETRY_POLICY: ConversationRetryPolicy = {
+	enabled: true,
+	maxRetries: 3,
+	baseDelayMs: 2000,
+	maxAgentDelayMs: 60000,
+};
+
+export const DEFAULT_COMPACTION_POLICY: CompactionPolicy = {
+	enabled: true,
+	reserveTokens: 16384,
+	keepRecentTokens: 20000,
+	backgroundTokens: 32768,
+};
+
+export const DEFAULT_PROGRESS_POLICY: ProgressPolicy = {
+	partialIntervalMs: 100,
+	outputIntervalMs: 100,
+};
+
+/** The reserved section key of the agent's `instructions`. */
+export const INSTRUCTIONS_KEY = "instructions";
+
+/** Built-in agent document; rewindable so forks start from the agent at their fork entry. */
+export const AgentDoc = defineDoc<AgentState>({
+	kind: "pi.agent",
+	version: 1,
+	scope: "conversation",
+	history: "rewindable",
+	fork: "asOf",
+	initial: () => ({}),
+	checkpointWhen: () => true,
+});
+
+/** Default `settings.contextRetentionMs`: ten minutes. */
+const DEFAULT_CONTEXT_RETENTION_MS = 600_000;
+
+/** Resolve the host settings: every field over its built-in default, object fields merged. */
+export function resolveSettings(settings: HarnessSettings | undefined): Settings {
+	const extensions = settings?.extensions;
+	return {
+		...(extensions === undefined ? {} : { extensions }),
+		stream: { ...settings?.stream },
+		retry: { ...DEFAULT_RETRY_POLICY, ...settings?.retry },
+		compaction: { ...DEFAULT_COMPACTION_POLICY, ...settings?.compaction },
+		// Field by field, so an explicitly undefined interval keeps its default.
+		progress: {
+			partialIntervalMs: settings?.progress?.partialIntervalMs ?? DEFAULT_PROGRESS_POLICY.partialIntervalMs,
+			outputIntervalMs: settings?.progress?.outputIntervalMs ?? DEFAULT_PROGRESS_POLICY.outputIntervalMs,
+		},
+		toolExecution: settings?.toolExecution ?? "parallel",
+		steeringMode: settings?.steeringMode ?? "one-at-a-time",
+		followUpMode: settings?.followUpMode ?? "one-at-a-time",
+		contextRetentionMs: settings?.contextRetentionMs ?? DEFAULT_CONTEXT_RETENTION_MS,
+		documentRetentionMs: documentRetention(settings?.documentRetentionMs),
+	};
+}
+
+/** Apply one change to `pi.agent`: a given field replaces the stored one, `null` clears it, `undefined` changes nothing. */
+export async function configure(tx: Tx, conversationId: ConversationId, change: AgentChange): Promise<void> {
+	const state = await tx.doc(AgentDoc, conversationId);
+	applyChange(state, change);
+}
+
+/**
+ * `addTools` of a tool round, applied to `tools` and `modelTools`, so the next request offers the tools: an array gets
+ * each name it lacks appended, and `{ remove }` loses the names. An unset `tools` already enables every tool, so nothing
+ * is written for it. An unset or object `modelTools` also gets each name appended to its `add`, which loads deferred
+ * tools and has no effect for others. By name, without the registry: a tool that is not registered now, such as one of
+ * an MCP server that is reconnecting, is offered once it is again.
+ */
+export async function addTools(tx: Tx, conversationId: ConversationId, added: readonly string[]): Promise<void> {
+	const state = await tx.doc(AgentDoc, conversationId);
+	const tools = state.tools;
+	if (Array.isArray(tools)) {
+		for (const name of added) if (!tools.includes(name)) tools.push(name);
+	} else if (tools?.remove.some((name) => added.includes(name))) {
+		state.tools = { remove: tools.remove.filter((name) => !added.includes(name)) };
+	}
+	const modelTools = state.modelTools;
+	if (Array.isArray(modelTools)) {
+		for (const name of added) if (!modelTools.includes(name)) modelTools.push(name);
+		return;
+	}
+	// Assigned whole; a document write of an equal value records no change.
+	const remove = (modelTools?.remove ?? []).filter((name) => !added.includes(name));
+	const add = [...(modelTools?.add ?? [])];
+	for (const name of added) if (!add.includes(name)) add.push(name);
+	state.modelTools = { ...(remove.length === 0 ? {} : { remove }), add };
+}
+
+function applyChange(state: Draft<AgentState>, change: AgentChange): void {
+	const set = <K extends keyof AgentState>(key: K, value: AgentState[K] | null | undefined) => {
+		if (value === undefined) return;
+		if (value === null) delete state[key];
+		else state[key] = value as Draft<AgentState>[K];
+	};
+	set("model", change.model === undefined || change.model === null ? change.model : { ...change.model });
+	set("thinkingLevel", change.thinkingLevel);
+	const extensions = change.extensions;
+	set(
+		"extensions",
+		extensions === undefined || extensions === null
+			? extensions
+			: isList(extensions)
+				? names(extensions)
+				: {
+						...(extensions.add === undefined ? {} : { add: names(extensions.add) }),
+						...(extensions.remove === undefined ? {} : { remove: names(extensions.remove) }),
+					},
+	);
+	const tools = change.tools;
+	set(
+		"tools",
+		tools === undefined || tools === null ? tools : isList(tools) ? names(tools) : { remove: names(tools.remove) },
+	);
+	const modelTools = change.modelTools;
+	set(
+		"modelTools",
+		modelTools === undefined || modelTools === null
+			? modelTools
+			: isList(modelTools)
+				? names(modelTools)
+				: {
+						...(modelTools.remove === undefined ? {} : { remove: names(modelTools.remove) }),
+						...(modelTools.add === undefined ? {} : { add: names(modelTools.add) }),
+					},
+	);
+	set("instructions", change.instructions);
+	set("cwd", change.cwd);
+}
+
+function isList<T>(value: readonly T[] | object): value is readonly T[] {
+	return Array.isArray(value);
+}
+
+function names(items: readonly { readonly name: string }[]): string[] {
+	return items.map((item) => item.name);
+}
+
+/**
+ * Built-in part of every Harness commit that creates or forks a conversation, for `pi.agent`: a fork keeps its `asOf`
+ * copy; a new task-owned conversation copies the stored agent of its owner task's conversation; a new ownerless one
+ * starts empty.
+ */
+export async function createAgent(tx: Tx, conversation: ConversationRecord): Promise<void> {
+	if (conversation.parent !== undefined) return;
+	const agent = await tx.doc(AgentDoc, conversation.id);
+	if (conversation.owner === undefined) return;
+	const owner = await tx.doc(AgentDoc, conversation.owner.conversationId);
+	Object.assign(agent, copyJson(owner) as AgentState);
+}
+
+/** Handlers of the selected extensions' hooks for a task name, in extension order. */
+export function agentHooks(agent: Agent, taskName: string): object[] {
+	const handlers: object[] = [];
+	for (const extension of agent.extensions) {
+		for (const hook of extension.hooks ?? []) if (hook.task === taskName) handlers.push(hook.handlers);
+	}
+	return handlers;
+}
+
+/**
+ * Resolve an agent from its stored state (absent: every field unset), a registry snapshot, and resolved settings. A
+ * wrapper that throws or renames drops its target and is reported; a wrapper without a target does nothing.
+ */
+export function resolveAgent<Tool extends ToolRegistration>(
+	state: Readonly<AgentState> | undefined,
+	snapshot: RegistrySnapshot<Tool>,
+	settings: Settings,
+	report: (error: unknown) => void,
+): Agent<Tool> {
+	const extensions = selectExtensions(state?.extensions, snapshot, settings);
+
+	const composed = new Map<string, Tool>();
+	for (const extension of extensions) for (const tool of extension.tools ?? []) composed.set(tool.name, tool);
+	const sections = new Map<string, PromptSection<Tool>>();
+	for (const extension of extensions) {
+		for (const section of extension.sections ?? []) sections.set(section.key, section);
+	}
+	for (const extension of extensions) {
+		for (const wrap of extension.wraps ?? []) {
+			if ("tool" in wrap)
+				applyWrap(
+					composed,
+					wrap.tool,
+					(tool) => wrap.wrap(tool),
+					(tool) => tool.name,
+					report,
+				);
+			else
+				applyWrap(
+					sections,
+					wrap.section,
+					(section) => wrap.wrap(section),
+					(section) => section.key,
+					report,
+				);
+		}
+	}
+
+	const enabled = filterTools([...composed.values()], state?.tools);
+	const callableBy = (caller: "model" | "tools") => (tool: Tool) =>
+		tool.callers === undefined || tool.callers.includes(caller);
+	const modelCallable = enabled.filter(callableBy("model"));
+	const tools = offeredTools(modelCallable, state?.modelTools);
+	const offered = new Set(tools);
+	const deferred = modelCallable.filter((tool) => tool.deferred === true && !offered.has(tool));
+	const callable = enabled.filter(callableBy("tools"));
+
+	const instructions = state?.instructions;
+	const agentSections = [...sections.values()];
+	if (instructions !== undefined) agentSections.push({ key: INSTRUCTIONS_KEY, render: () => instructions });
+
+	const agent: Agent<Tool> = {
+		...(state?.model === undefined ? {} : { model: state.model }),
+		thinkingLevel: state?.thinkingLevel ?? "off",
+		extensions,
+		tools,
+		callable,
+		deferred,
+		sections: agentSections,
+		...(instructions === undefined ? {} : { instructions }),
+		...(state?.cwd === undefined ? {} : { cwd: state.cwd }),
+	};
+	return agent;
+}
+
+/** The tools offered to the model, as `AgentState.modelTools` describes. */
+function offeredTools<Tool extends ToolRegistration>(tools: Tool[], filter: AgentState["modelTools"]): Tool[] {
+	if (Array.isArray(filter)) return filterTools(tools, filter);
+	const removed = new Set(filter?.remove ?? []);
+	const kept = tools.filter((tool) => tool.deferred !== true && !removed.has(tool.name));
+	// Loaded deferred tools follow in load order, so loading one appends to the declared tools.
+	const byName = new Map(tools.map((tool) => [tool.name, tool]));
+	const loaded = [...new Set(filter?.add ?? [])].flatMap((name) => {
+		const tool = byName.get(name);
+		return tool?.deferred === true && !removed.has(name) ? [tool] : [];
+	});
+	return [...kept, ...loaded];
+}
+
+/** `tools` through a stored filter: an array selects exactly its names, in its order; `{ remove }` drops names. */
+function filterTools<Tool extends ToolRegistration>(tools: Tool[], filter: AgentState["tools"]): Tool[] {
+	if (filter === undefined) return tools;
+	if (Array.isArray(filter)) {
+		const byName = new Map(tools.map((tool) => [tool.name, tool]));
+		return [...new Set(filter)].flatMap((name) => {
+			const tool = byName.get(name);
+			return tool === undefined ? [] : [tool];
+		});
+	}
+	const removed = new Set((filter as { remove: string[] }).remove);
+	return tools.filter((tool) => !removed.has(tool.name));
+}
+
+/** Selected installed extensions: the stored array, or the default selection edited by `{ add, remove }`. */
+function selectExtensions<Tool extends ToolRegistration>(
+	stored: AgentState["extensions"],
+	snapshot: RegistrySnapshot<Tool>,
+	settings: Settings,
+): Extension<Tool>[] {
+	let selected: string[];
+	if (Array.isArray(stored)) selected = stored;
+	else {
+		const base = settings.extensions?.map((extension) => extension.name) ?? snapshot.installed().map((e) => e.name);
+		const edit = stored as { add?: string[]; remove?: string[] } | undefined;
+		const removed = new Set(edit?.remove ?? []);
+		selected = [...base, ...(edit?.add ?? [])].filter((name) => !removed.has(name));
+	}
+	const extensions: Extension<Tool>[] = [];
+	for (const name of new Set(selected)) {
+		const extension = snapshot.extension(name);
+		if (extension !== undefined) extensions.push(extension);
+	}
+	return extensions;
+}
+
+function applyWrap<T>(
+	items: Map<string, T>,
+	target: string,
+	wrap: (item: T) => T,
+	nameOf: (item: T) => string,
+	report: (error: unknown) => void,
+): void {
+	const item = items.get(target);
+	if (item === undefined) return;
+	try {
+		const wrapped = wrap(item);
+		if (nameOf(wrapped) !== target) throw new Error(`Wrapper renamed ${target} to ${nameOf(wrapped)}`);
+		items.set(target, wrapped);
+	} catch (error) {
+		items.delete(target);
+		report(error);
+	}
+}

third_party/pi/packages/durable/src/harness/compaction.ts

@@ -0,0 +1,464 @@
+import type { Context, Draft } from "@earendil-works/chord";
+import type { AssistantMessage, Message, ModelThinkingLevel, SimpleStreamOptions } from "@earendil-works/pi-ai";
+import { calculateContextTokens, estimateMessageTokens } from "@earendil-works/pi-ai/utils/estimate";
+import { isRetryableAssistantError, retryDelayMs } from "@earendil-works/pi-ai/utils/retry";
+import { CompactionEntry } from "../entries.ts";
+import { defineTask } from "../tasks.ts";
+import type {
+	ConversationId,
+	EntryDraft,
+	EntryId,
+	NextTaskState,
+	RunningTask,
+	TaskId,
+	TaskRuntime,
+	Tx,
+} from "../types.ts";
+import { orderToolResults } from "./context.ts";
+import {
+	addCompactionStatus,
+	compactionStatus,
+	failCompactionStatus,
+	LiveDoc,
+	type LiveState,
+	removeCompactionStatus,
+} from "./live.ts";
+import { ensureProviderSessionId } from "./provider.ts";
+import { admitSubmission } from "./submissions.ts";
+import type {
+	CompactionHooks,
+	CompactionReason,
+	CompactionResult,
+	ContextView,
+	ConversationStreamOptions,
+	ModelRef,
+} from "./types.ts";
+import { recordUsage } from "./usage.ts";
+
+export type CompactionInput = { reason: CompactionReason; instructions?: string };
+
+/** The pinned summarization request. */
+export type SummaryRequest = {
+	attempt: number;
+	model: ModelRef;
+	thinkingLevel: ModelThinkingLevel;
+	streamOptions: ConversationStreamOptions;
+	maxTokens: number;
+	/** Newest entry of the context the range was selected from. */
+	tail: EntryId;
+	/** First entry kept verbatim; the summary's `head`. */
+	firstKept: EntryId;
+};
+
+export type CompactionCheckpoint =
+	| { phase: "select" }
+	| ({ phase: "summarize" } & SummaryRequest)
+	| ({ phase: "retry"; until: number } & SummaryRequest);
+
+type Runtime = TaskRuntime<CompactionInput, CompactionCheckpoint, CompactionResult, CompactionHooks>;
+type Next = NextTaskState<CompactionCheckpoint, CompactionResult>;
+
+/** Longest tool result text a serialized summary source keeps. */
+const TOOL_RESULT_MAX_CHARS = 2000;
+
+const SUMMARY_PREFIX =
+	"The conversation history before this point was compacted into the following summary:\n\n<summary>\n";
+const SUMMARY_SUFFIX = "\n</summary>";
+
+const SUMMARIZATION_SYSTEM_PROMPT = `You are a context summarization assistant. Your task is to read a conversation between a user and an AI assistant, then produce a structured summary following the exact format specified.
+
+Do NOT continue the conversation. Do NOT respond to any questions in the conversation. ONLY output the structured summary.`;
+
+const SUMMARIZATION_PROMPT = `The messages above are a conversation to summarize. Create a structured context checkpoint summary that another LLM will use to continue the work. If the conversation starts with an earlier summary, preserve its information and fold the newer messages into it.
+
+Use this EXACT format:
+
+## Goal
+[What is the user trying to accomplish? Can be multiple items if the session covers different tasks.]
+
+## Constraints & Preferences
+- [Any constraints, preferences, or requirements mentioned by user]
+- [Or "(none)" if none were mentioned]
+
+## Progress
+### Done
+- [x] [Completed tasks/changes]
+
+### In Progress
+- [ ] [Current work]
+
+### Blocked
+- [Issues preventing progress, if any]
+
+## Key Decisions
+- **[Decision]**: [Brief rationale]
+
+## Next Steps
+1. [Ordered list of what should happen next]
+
+## Critical Context
+- [Any data, examples, or references needed to continue]
+- [Or "(none)" if not applicable]
+
+Keep each section concise. Preserve exact file paths, function names, and error messages.`;
+
+/**
+ * Built-in compaction task (spec §8.7): select an old prefix of the model context, summarize it, and place a summary
+ * entry whose `head` is the first kept entry. A compaction the generation owns blocks it and appends directly; a
+ * conversation-owned one places its summary through a write submission.
+ */
+export const CompactionTask = defineTask<CompactionInput, CompactionCheckpoint, CompactionResult, CompactionHooks>({
+	name: "pi.compaction",
+	version: 1,
+	initial: () => ({ phase: "select" }),
+	phases: {
+		select: async (task, runtime, context) => {
+			const { conversationId } = runtime;
+			const agent = await runtime.agent(context);
+			const settings = runtime.settings;
+			const ref = agent.model;
+			const model = ref === undefined ? undefined : runtime.models.getModel(ref.provider, ref.modelId);
+			if (ref === undefined || model === undefined) return failNoModel(runtime, ref, context);
+			const policy = settings.compaction;
+			const view = await runtime.context(conversationId, context);
+			const cut = selectCut(view, policy.keepRecentTokens);
+			if (cut === undefined) return complete(runtime, context);
+			const firstKept = view.entries[cut]!.id;
+			const { reason, instructions } = task.input;
+			const compaction = {
+				reason,
+				entries: view.entries.slice(0, cut),
+				messages: summarizedMessages(view, cut),
+				firstKept,
+				...(instructions === undefined ? {} : { instructions }),
+			};
+			let decision: { readonly decline: true } | { readonly summary: string } | undefined;
+			await runtime.hooks.each("beforeCompact", async (hook) => {
+				if (decision === undefined) decision = await hook(compaction, runtime, context);
+			});
+			if (decision !== undefined && "decline" in decision) return complete(runtime, context);
+			if (decision !== undefined) return place(runtime, firstKept, decision.summary, context);
+			const request: SummaryRequest = {
+				attempt: 1,
+				model: ref,
+				thinkingLevel: agent.thinkingLevel,
+				streamOptions: settings.stream,
+				maxTokens: Math.min(
+					Math.floor(0.8 * policy.reserveTokens),
+					model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
+				),
+				tail: view.entries.reduce((tail, entry) => (entry.id > tail ? entry.id : tail), firstKept),
+				firstKept,
+			};
+			await runtime.commit(() => ({ status: "running", checkpoint: { phase: "summarize", ...request } }), context);
+		},
+		summarize: async (task, runtime, context) => {
+			const { phase: _, ...request } = task.state.checkpoint;
+			const { model: ref, thinkingLevel, streamOptions, maxTokens, tail, firstKept, attempt } = request;
+			const model = runtime.models.getModel(ref.provider, ref.modelId);
+			if (model === undefined) return failNoModel(runtime, ref, context);
+			// The context at `tail` is immutable, so this is the range `select` chose.
+			const view = await runtime.context(runtime.conversationId, context, { at: tail });
+			const cut = view.entries.findIndex((entry) => entry.id === firstKept);
+			const now = runtime.now();
+			const messages: Message[] = [
+				{ role: "system", content: SUMMARIZATION_SYSTEM_PROMPT, timestamp: now },
+				{
+					role: "user",
+					content: [{ type: "text", text: summaryPrompt(summarizedMessages(view, cut), task.input.instructions) }],
+					timestamp: now,
+				},
+			];
+			const { deferred: _deferred, ...forwarded } = streamOptions;
+			const options: SimpleStreamOptions = {
+				...forwarded,
+				cacheRetention: "none",
+				maxTokens,
+				signal: runtime.signal,
+				sessionId: await ensureProviderSessionId(runtime, context),
+				...(thinkingLevel === "off" ? {} : { reasoning: thinkingLevel }),
+			};
+			const message = await runtime.models.completeSimple(model, { messages }, options);
+			// An abort mark or close: the abort invocation or the reopened task handles the committed state.
+			runtime.signal.throwIfAborted();
+			const summary = summaryText(message);
+			const policy = runtime.settings.retry;
+			const retry =
+				message.stopReason === "error" &&
+				isRetryableAssistantError(message) &&
+				policy.enabled &&
+				attempt <= policy.maxRetries;
+			const until = retry ? runtime.now() + retryDelayMs(policy, attempt) : 0;
+			await runtime.commit(async (tx, current): Promise<Next> => {
+				await recordUsage(
+					tx,
+					runtime.conversationId,
+					"models",
+					`${message.provider}/${message.model}`,
+					message.usage,
+				);
+				const live = await tx.doc(LiveDoc, runtime.conversationId);
+				if (summary !== undefined) return placeSummary(tx, runtime, current, live, firstKept, summary);
+				if (retry) {
+					const status = compactionStatus(live, runtime.taskId);
+					if (status !== undefined) status.retry = { at: until, error: message.errorMessage ?? "" };
+					return { status: "running", checkpoint: { phase: "retry", ...request, until } };
+				}
+				const text = summaryFailure(message);
+				failCompactionStatus(live, runtime.taskId, text);
+				return {
+					status: "terminal",
+					outcome: { status: "failed", error: { message: text, detail: { reason: "model_error" } } },
+				};
+			}, context);
+		},
+		retry: async (task, runtime, context) => {
+			const { phase: _, until, ...request } = task.state.checkpoint;
+			await runtime.sleep(until, context);
+			const attempt = request.attempt + 1;
+			await runtime.commit(async (tx) => {
+				const status = compactionStatus(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId);
+				if (status !== undefined) {
+					status.attempt = attempt;
+					delete status.retry;
+				}
+				return { status: "running", checkpoint: { phase: "summarize", ...request, attempt } };
+			}, context);
+		},
+	},
+	abort: async (_task, runtime, context) => {
+		await runtime.commit(async (tx) => {
+			removeCompactionStatus(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId);
+			return { status: "terminal", outcome: { status: "aborted" } };
+		}, context);
+	},
+});
+
+/**
+ * Create a compaction task with its status in this commit. `owner` is the generation that waits for it (a blocking
+ * compaction); without one it is conversation-owned, and `background` unless it is manual.
+ */
+export async function createCompaction(
+	tx: Tx,
+	conversationId: ConversationId,
+	input: CompactionInput,
+	owner?: TaskId,
+): Promise<TaskId<CompactionResult>> {
+	const ownership =
+		owner === undefined ? ({ kind: "conversation" } as const) : { kind: "task" as const, taskId: owner };
+	const background = owner === undefined && input.reason !== "manual";
+	const taskId = await tx.createTask(CompactionTask, input, { ownership, conversationId, background });
+	const status = { taskId, reason: input.reason, blocking: owner !== undefined, attempt: 1 };
+	addCompactionStatus(await tx.doc(LiveDoc, conversationId), status);
+	return taskId;
+}
+
+/**
+ * Index in `view.entries` of the first entry a summary keeps, or `undefined` when there is nothing to compact
+ * (spec §8.7). Walks back from the tail until `keepRecentTokens` are kept, then cuts at the first candidate at or after
+ * that entry: an entry whose contribution starts with a user or assistant message, never a tool result, and never a
+ * user entry that a result of the preceding assistant's calls still follows.
+ */
+export function selectCut(view: ContextView, keepRecentTokens: number): number | undefined {
+	const { contributions } = view;
+	const start = view.head === undefined ? 0 : 1;
+	const candidates: number[] = [];
+	for (let index = start; index < contributions.length; index++) {
+		if (isCandidate(contributions, index)) candidates.push(index);
+	}
+	let kept = 0;
+	let cut: number | undefined;
+	for (let index = contributions.length - 1; index >= start; index--) {
+		for (const message of contributions[index]!) kept += estimateMessageTokens(message);
+		if (kept < keepRecentTokens) continue;
+		cut = candidates.find((candidate) => candidate >= index) ?? candidates.at(-1);
+		break;
+	}
+	if (cut === undefined) return undefined;
+	for (let index = start; index < cut; index++) if (contributions[index]!.length > 0) return cut;
+	return undefined;
+}
+
+function isCandidate(contributions: ContextView["contributions"], index: number): boolean {
+	const first = contributions[index]![0];
+	if (first?.role === "assistant") return true;
+	if (first?.role !== "user") return false;
+	// A result of the preceding assistant's calls that follows this entry, before the next assistant, belongs before it.
+	let calls: ReadonlySet<string> = new Set();
+	for (let before = index - 1; before >= 0; before--) {
+		const assistant = contributions[before]!.findLast((message) => message.role === "assistant");
+		if (assistant === undefined) continue;
+		calls = new Set(assistant.content.flatMap((content) => (content.type === "toolCall" ? [content.id] : [])));
+		break;
+	}
+	if (calls.size === 0) return true;
+	for (let after = index; after < contributions.length; after++) {
+		for (const [position, message] of contributions[after]!.entries()) {
+			if (message.role === "assistant" && (after > index || position > 0)) return true;
+			if (message.role === "toolResult" && calls.has(message.toolCallId)) return false;
+		}
+	}
+	return true;
+}
+
+/** Model messages of the entries before `cut`: the head marker first, ordered like model context (spec §2.1). */
+export function summarizedMessages(view: ContextView, cut: number): Message[] {
+	return orderToolResults(view.contributions.slice(0, cut).flat());
+}
+
+/**
+ * Size of a request over `view` followed by `extra` (spec §8.3): the usage of the newest assistant appended after the
+ * head marker, whose request included the marker, plus estimates of the messages after it; without one, estimates of
+ * every message.
+ */
+export function estimateContext(view: ContextView, extra: readonly Message[]): number {
+	let measured: AssistantMessage | undefined;
+	const after = view.head?.id ?? Number.NEGATIVE_INFINITY;
+	for (let index = view.entries.length - 1; index >= 0 && measured === undefined; index--) {
+		if (view.entries[index]!.id <= after) continue;
+		measured = view.contributions[index]!.findLast(
+			(message): message is AssistantMessage =>
+				message.role === "assistant" && calculateContextTokens(message.usage) > 0,
+		);
+	}
+	const from = measured === undefined ? 0 : view.messages.lastIndexOf(measured) + 1;
+	let tokens = measured === undefined ? 0 : calculateContextTokens(measured.usage);
+	for (const message of view.messages.slice(from)) tokens += estimateMessageTokens(message);
+	for (const message of extra) tokens += estimateMessageTokens(message);
+	return tokens;
+}
+
+/** The summary of a clean `stop` with text and no tool call; anything else is not a summary. */
+function summaryText(message: AssistantMessage): string | undefined {
+	if (message.stopReason !== "stop" || message.content.some((content) => content.type === "toolCall"))
+		return undefined;
+	const text = message.content
+		.flatMap((content) => (content.type === "text" ? [content.text] : []))
+		.join("\n")
+		.trim();
+	return text.length === 0 ? undefined : text;
+}
+
+function summaryFailure(message: AssistantMessage): string {
+	if (message.stopReason === "error" || message.stopReason === "aborted") {
+		return `Summarization failed: ${message.errorMessage ?? message.stopReason}`;
+	}
+	if (message.stopReason === "length") return "Summarization hit the token limit; the summary is incomplete";
+	if (message.content.some((content) => content.type === "toolCall")) return "Summarization attempted to call a tool";
+	return "Summarization produced no text";
+}
+
+/** The summarizer's user message: the serialized conversation, the prompt, and any instructions. */
+function summaryPrompt(messages: readonly Message[], instructions: string | undefined): string {
+	const focus = instructions === undefined ? "" : `\n\nAdditional focus: ${instructions}`;
+	return `<conversation>\n${serializeConversation(messages)}\n</conversation>\n\n${SUMMARIZATION_PROMPT}${focus}`;
+}
+
+/**
+ * Messages as plain text, so the summarizer reads a transcript instead of continuing it. System messages are omitted.
+ * Thinking is omitted too: providers return summaries of the real reasoning, and replayed thinking is often not part of
+ * the live context (a block that does not match its request is dropped), so including it can make the request larger
+ * than the context it summarizes. Visible text, tool calls, and tool results record what happened.
+ */
+export function serializeConversation(messages: readonly Message[]): string {
+	const parts: string[] = [];
+	for (const message of messages) {
+		if (message.role === "user") {
+			const text = contentText(message.content);
+			if (text.length > 0) parts.push(`[User]: ${text}`);
+		} else if (message.role === "assistant") {
+			const text = message.content.flatMap((content) => (content.type === "text" ? [content.text] : []));
+			const calls = message.content.flatMap((content) =>
+				content.type === "toolCall"
+					? [
+							`${content.name}(${Object.entries(content.arguments)
+								.map(([key, value]) => `${key}=${JSON.stringify(value)}`)
+								.join(", ")})`,
+						]
+					: [],
+			);
+			if (text.length > 0) parts.push(`[Assistant]: ${text.join("\n")}`);
+			if (calls.length > 0) parts.push(`[Assistant tool calls]: ${calls.join("; ")}`);
+		} else if (message.role === "toolResult") {
+			const text = contentText(message.content);
+			if (text.length > 0) parts.push(`[Tool result]: ${truncate(text, TOOL_RESULT_MAX_CHARS)}`);
+		}
+	}
+	return parts.join("\n\n");
+}
+
+function contentText(content: string | readonly { readonly type: string; readonly text?: string }[]): string {
+	if (typeof content === "string") return content;
+	return content
+		.flatMap((block) => (block.type === "text" && block.text !== undefined ? [block.text] : []))
+		.join("\n");
+}
+
+function truncate(text: string, maxChars: number): string {
+	if (text.length <= maxChars) return text;
+	return `${text.slice(0, maxChars)}\n\n[... ${text.length - maxChars} more characters truncated]`;
+}
+
+/** Place a summary supplied by a hook in its own commit. */
+async function place(runtime: Runtime, firstKept: EntryId, summary: string, context: Context): Promise<void> {
+	await runtime.commit(
+		async (tx, current) =>
+			placeSummary(tx, runtime, current, await tx.doc(LiveDoc, runtime.conversationId), firstKept, summary),
+		context,
+	);
+}
+
+/**
+ * Place the summary entry and complete (spec §8.7). A blocking compaction, owned by its generation, appends it: the
+ * generation holds the run and waits. A conversation-owned one admits it as a write submission, placed at once when
+ * idle, otherwise at the next boundary, or settled `stale`.
+ * REMINDER: nothing else may append to a busy conversation, so every non-blocking summary goes through admission.
+ */
+async function placeSummary(
+	tx: Tx,
+	runtime: Runtime,
+	current: RunningTask<CompactionInput, CompactionCheckpoint, CompactionResult>,
+	live: Draft<LiveState>,
+	firstKept: EntryId,
+	summary: string,
+): Promise<Next> {
+	removeCompactionStatus(live, runtime.taskId);
+	delete live.compactionFailure;
+	const text = `${SUMMARY_PREFIX}${summary}${SUMMARY_SUFFIX}`;
+	const entry = {
+		kind: CompactionEntry.kind,
+		head: firstKept,
+		model: [{ role: "user", content: [{ type: "text", text }], timestamp: runtime.now() }],
+		data: { reason: current.input.reason },
+	} satisfies EntryDraft;
+	const result: CompactionResult =
+		current.owner === undefined
+			? {
+					submissionId: await admitSubmission(
+						tx,
+						runtime.conversationId,
+						{ type: "write", requestId: `compaction:${runtime.taskId}`, entry },
+						runtime.now(),
+						runtime.settings,
+					),
+				}
+			: { entryId: (await tx.appendEntry(runtime.conversationId, entry)).id };
+	return { status: "terminal", outcome: { status: "completed", result } };
+}
+
+/** Remove the status and complete without a summary. */
+async function complete(runtime: Runtime, context: Context): Promise<void> {
+	await runtime.commit(async (tx) => {
+		removeCompactionStatus(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId);
+		return { status: "terminal", outcome: { status: "completed", result: {} } };
+	}, context);
+}
+
+async function failNoModel(runtime: Runtime, ref: ModelRef | undefined, context: Context): Promise<void> {
+	const message =
+		ref === undefined ? "No model is configured" : `Model ${ref.provider}/${ref.modelId} is not available`;
+	await runtime.commit(async (tx) => {
+		failCompactionStatus(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId, message);
+		return { status: "terminal", outcome: { status: "failed", error: { message, detail: { reason: "no_model" } } } };
+	}, context);
+}

third_party/pi/packages/durable/src/harness/context.ts

@@ -0,0 +1,333 @@
+import type { Context } from "@earendil-works/chord";
+import type { AssistantMessage, Message, ToolCall, ToolResultMessage } from "@earendil-works/pi-ai";
+import { SystemEntry } from "../entries.ts";
+import { idFromNumber } from "../ids.ts";
+import type { SessionImpl } from "../session/session.ts";
+import type { ContextEdit, ConversationId, Cursor, EntryId, EntryQuery, EntryRecord, Storage } from "../types.ts";
+import type { ContextView } from "./types.ts";
+
+const SCAN_PAGE_SIZE = 256;
+const EXCLUDED_STOP_REASONS: ReadonlySet<AssistantMessage["stopReason"]> = new Set(["aborted", "error", "deferred"]);
+const MISSING_RESULT_TEXT = "Tool result unavailable: history ends before this call completed.";
+
+/** Head marker and newest visible entry that fix one committed context range. */
+export type ContextBounds = {
+	readonly head: (EntryRecord & { readonly head: EntryId }) | undefined;
+	readonly tail: EntryId;
+};
+
+/**
+ * Capture the bounds of the current context, or of the context cut off at the visible entry `at`, with two O(1)
+ * reads. Run this on the Session line; entries at or below the tail are immutable, so `deriveContext()` can then scan
+ * them off the line.
+ */
+export async function captureContextBounds(
+	storage: Storage,
+	conversationId: ConversationId,
+	context: Context,
+	at?: EntryId,
+): Promise<ContextBounds | undefined> {
+	let tail: EntryId | undefined;
+	if (at === undefined) {
+		tail = (await storage.scanEntries({ conversationId }, 1, undefined, context)).items[0]?.id;
+		if (tail === undefined) return undefined;
+	} else {
+		if ((await storage.entry(conversationId, at, context)) === undefined) {
+			throw new Error(`Entry ${at} is not visible from conversation ${conversationId}`);
+		}
+		tail = at;
+	}
+	return { head: await storage.findLatestHeadMarker(conversationId, tail, context), tail };
+}
+
+/**
+ * One context read: the visible entries it scanned, from the head marker's head, or transcript start, through
+ * `bounds.tail`, oldest first, and the view derived from them. Entries at or below the tail never change, so a later
+ * read with the same head marker extends both.
+ */
+export type ContextRange = {
+	readonly bounds: ContextBounds;
+	readonly entries: readonly EntryRecord[];
+	readonly view: ContextView;
+	/** Targets of the edits in `entries`. */
+	readonly edited: ReadonlySet<EntryId>;
+	/**
+	 * `view.messages` from the contributed messages before the last assistant message. Tool results are ordered within
+	 * the messages up to the next assistant message, so later entries cannot change these.
+	 */
+	readonly settled: readonly Message[];
+	/** Contributed messages from the last assistant message on, before tool result ordering. */
+	readonly open: readonly Message[];
+	/** The baseline's system message; see `findBaseline()`. */
+	readonly baseline: Baseline;
+};
+
+/** A baseline's system message, `null` when the range cannot have one, or `undefined` while a later entry may be one. */
+type Baseline = Message | null | undefined;
+
+/**
+ * `readContext()` that reuses `previous`, an earlier range of the same conversation: with the same head marker, only
+ * entries after its tail are scanned. Returns the view and the range to pass to the next read. A range outlives the
+ * read, so its entries are frozen like `MemoryStorage` records, and the returned view has its own arrays.
+ */
+export async function readContextFrom(
+	session: SessionImpl,
+	storage: Storage,
+	conversationId: ConversationId,
+	context: Context,
+	at: EntryId | undefined,
+	previous: ContextRange | undefined,
+): Promise<{ readonly view: ContextView; readonly range: ContextRange | undefined }> {
+	const bounds = await session.readOnLine(() => captureContextBounds(storage, conversationId, context, at));
+	if (bounds === undefined) return { view: emptyView(), range: undefined };
+	freezeJson(bounds.head);
+	let range: ContextRange;
+	if (previous === undefined || previous.bounds.head?.id !== bounds.head?.id) {
+		range = deriveRange(bounds, freezeJson(await scanRange(storage, conversationId, rangeQuery(bounds), context)));
+	} else if (bounds.tail === previous.bounds.tail) {
+		range = previous;
+	} else if (bounds.tail < previous.bounds.tail) {
+		range = deriveRange(
+			bounds,
+			previous.entries.filter((entry) => entry.id <= bounds.tail),
+		);
+	} else {
+		const minEntryId = idFromNumber<EntryId>(previous.bounds.tail + 1);
+		const added = await scanRange(storage, conversationId, { minEntryId, maxEntryId: bounds.tail }, context);
+		range = extendRange(previous, bounds, freezeJson(added));
+	}
+	const { view } = range;
+	return {
+		view: {
+			...view,
+			entries: [...view.entries],
+			contributions: [...view.contributions],
+			messages: [...view.messages],
+		},
+		range,
+	};
+}
+
+/** Committed context of one conversation: bounds captured on the Session line, entries derived off it. */
+export async function readContext(
+	session: SessionImpl,
+	storage: Storage,
+	conversationId: ConversationId,
+	context: Context,
+	at?: EntryId,
+): Promise<ContextView> {
+	const bounds = await session.readOnLine(() => captureContextBounds(storage, conversationId, context, at));
+	return deriveContext(storage, conversationId, bounds, context);
+}
+
+/**
+ * Derive the active transcript and model context of one conversation within captured bounds.
+ *
+ * H = newest visible head marker; the range runs from `H.head` (or transcript start) through the tail. Per target,
+ * the newest edit in the range wins. Context entries are H followed by the range's non-head entries.
+ */
+export async function deriveContext(
+	storage: Storage,
+	conversationId: ConversationId,
+	bounds: ContextBounds | undefined,
+	context: Context,
+): Promise<ContextView> {
+	if (bounds === undefined) return emptyView();
+	return deriveRange(bounds, await scanRange(storage, conversationId, rangeQuery(bounds), context)).view;
+}
+
+function emptyView(): ContextView {
+	return { head: undefined, entries: [], contributions: [], messages: [] };
+}
+
+/** Derive the context view from the entries scanned within `bounds`. */
+function deriveRange(bounds: ContextBounds, entries: readonly EntryRecord[]): ContextRange {
+	const edits = new Map<EntryId, ContextEdit>();
+	// Edits of every entry in the range count, including older head markers that `selectActive()` drops.
+	for (const entry of entries) for (const edit of entry.edits ?? []) edits.set(edit.target, edit);
+	const active = selectActive(bounds.head, entries);
+	const contributions = active.map((entry) => contribute(entry, edits.get(entry.id)));
+	const { settled, open } = settle([], contributions.flat());
+	const baseline = findBaseline(bounds.head, active);
+	const messages = leadWithSystem([...settled, ...orderToolResults(open)], baseline);
+	const view = { head: bounds.head, entries: active, contributions, messages };
+	return { bounds, entries, view, edited: new Set(edits.keys()), settled, open, baseline };
+}
+
+/**
+ * `previous` extended by the entries `added` after its tail under the same head marker: only their contributions and
+ * the open messages are derived. An added edit can change an earlier entry, so it derives the whole range again.
+ */
+function extendRange(previous: ContextRange, bounds: ContextBounds, added: readonly EntryRecord[]): ContextRange {
+	const entries = [...previous.entries, ...added];
+	if (added.some((entry) => entry.edits !== undefined || entry.head !== undefined || previous.edited.has(entry.id))) {
+		return deriveRange(bounds, entries);
+	}
+	const contributions = added.map((entry) => contribute(entry, undefined));
+	const { settled, open } = settle(previous.settled, [...previous.open, ...contributions.flat()]);
+	const baseline = previous.baseline !== undefined ? previous.baseline : findBaseline(bounds.head, added);
+	const view = {
+		head: bounds.head,
+		entries: [...previous.view.entries, ...added],
+		contributions: [...previous.view.contributions, ...contributions],
+		messages: leadWithSystem([...settled, ...orderToolResults(open)], baseline),
+	};
+	return { bounds, entries, view, edited: previous.edited, settled, open, baseline };
+}
+
+/**
+ * The baseline among `entries`, oldest first: the system message of the first `pi.system` entry after the head
+ * marker, or of the transcript without one, when generation wrote it with every section and tool in effect. After a
+ * head marker it always does, and omits the retained `pi.system` entries. Without one it does when it saw no system
+ * message before, which holds unless an earlier entry carries one, even if an edit omits it later. An edit that targets
+ * the baseline contributes other messages, or none, so none leads; a later entry does not take its place.
+ */
+function findBaseline(head: ContextBounds["head"], entries: readonly EntryRecord[]): Baseline {
+	for (const entry of entries) {
+		if (head !== undefined && entry.id <= head.id) continue;
+		if (SystemEntry.is(entry)) return entry.model?.find((message) => message.role === "system") ?? null;
+		if (head === undefined && carriesSystemMessage(entry)) return null;
+	}
+	return undefined;
+}
+
+/** Whether an entry's own messages, or the replacement messages of its edits, include a system message. */
+function carriesSystemMessage(entry: EntryRecord): boolean {
+	const isSystem = (message: Message) => message.role === "system";
+	return (
+		(entry.model?.some(isSystem) ?? false) ||
+		(entry.edits?.some((edit) => edit.action === "replace" && edit.messages.some(isSystem)) ?? false)
+	);
+}
+
+/**
+ * Move the first system message to the front when only user messages precede it, or when it is the baseline's. A run's
+ * input is committed before generation renders the system prompt, so a transcript starts with the input followed by
+ * the baseline, and the range after a compaction with the kept history, assistant messages included. Providers treat
+ * only a leading system message as the initial prompt and tool set; without it, a later tool change rewrites the
+ * request's tool list and invalidates the whole prompt cache. Only the first system message moves, so replaying
+ * system messages in order gives the same sections and tools. Any other system message, before an assistant message,
+ * stays where it was committed: moving it would change what it means to the model.
+ */
+function leadWithSystem(messages: Message[], baseline: Baseline): Message[] {
+	const index = messages.findIndex((message) => message.role === "system");
+	if (index <= 0) return messages;
+	const system = messages[index]!;
+	if (system !== baseline && messages.slice(0, index).some((message) => message.role !== "user")) {
+		return messages;
+	}
+	return [system, ...messages.slice(0, index), ...messages.slice(index + 1)];
+}
+
+/** One active entry's model messages after its edit and excluded stop reasons, before tool result ordering. */
+function contribute(entry: EntryRecord, edit: ContextEdit | undefined): readonly Message[] {
+	if (edit?.action === "omit") return Object.freeze([]);
+	const contributed = edit?.action === "replace" ? edit.messages : (entry.model ?? []);
+	return Object.freeze(
+		contributed.filter((message) => message.role !== "assistant" || !EXCLUDED_STOP_REASONS.has(message.stopReason)),
+	);
+}
+
+/** Objects `freezeJson()` froze with everything in them. `Object.isFrozen()` holds for shallow freezes too. */
+const deeplyFrozen = new WeakSet<object>();
+
+/** Freeze a JSON value and everything in it. */
+function freezeJson<T>(value: T): T {
+	if (value === null || typeof value !== "object" || deeplyFrozen.has(value)) return value;
+	for (const child of Object.values(value)) freezeJson(child);
+	Object.freeze(value);
+	deeplyFrozen.add(value);
+	return value;
+}
+
+/**
+ * Move the ordered messages before the last assistant message of `open` to `settled`. `orderToolResults()` of a
+ * sequence equals the concatenation over its parts when each later part starts with an assistant message.
+ */
+function settle(
+	settled: readonly Message[],
+	open: readonly Message[],
+): { readonly settled: readonly Message[]; readonly open: readonly Message[] } {
+	const last = open.findLastIndex((message) => message.role === "assistant");
+	if (last <= 0) return { settled, open };
+	return { settled: [...settled, ...orderToolResults(open.slice(0, last))], open: open.slice(last) };
+}
+
+/** The raw active entries within captured bounds, without deriving model context. */
+export async function activeEntries(
+	storage: Storage,
+	conversationId: ConversationId,
+	bounds: ContextBounds | undefined,
+	context: Context,
+): Promise<readonly EntryRecord[]> {
+	if (bounds === undefined) return [];
+	return selectActive(bounds.head, await scanRange(storage, conversationId, rangeQuery(bounds), context));
+}
+
+/** The visible range of `bounds`: from the head marker's head, or transcript start, through the tail. */
+function rangeQuery(bounds: ContextBounds): Omit<EntryQuery, "conversationId"> {
+	const head = bounds.head;
+	return head === undefined ? { maxEntryId: bounds.tail } : { minEntryId: head.head, maxEntryId: bounds.tail };
+}
+
+/** Visible entries within `range`, oldest first. */
+async function scanRange(
+	storage: Storage,
+	conversationId: ConversationId,
+	range: Omit<EntryQuery, "conversationId">,
+	context: Context,
+): Promise<EntryRecord[]> {
+	const entries: EntryRecord[] = [];
+	let cursor: Cursor | undefined;
+	do {
+		const page = await storage.scanEntries({ conversationId, ...range }, SCAN_PAGE_SIZE, cursor, context);
+		entries.push(...page.items);
+		cursor = page.next;
+	} while (cursor !== undefined);
+	return entries.reverse();
+}
+
+/** The head marker followed by the range's non-head entries, or the whole range without a marker. */
+function selectActive(head: ContextBounds["head"], range: readonly EntryRecord[]): readonly EntryRecord[] {
+	return head === undefined ? range : [head, ...range.filter((entry) => entry.head === undefined)];
+}
+
+/**
+ * Place each assistant's tool results directly after it in call order. Results are taken from the messages before
+ * the next assistant; a missing result is synthesized and unmatched results are dropped.
+ */
+export function orderToolResults(messages: readonly Message[]): Message[] {
+	const ordered: Message[] = [];
+	for (let index = 0; index < messages.length; index++) {
+		const message = messages[index]!;
+		if (message.role === "toolResult") continue;
+		ordered.push(message);
+		if (message.role !== "assistant") continue;
+		const calls = message.content.filter((content): content is ToolCall => content.type === "toolCall");
+		if (calls.length === 0) continue;
+		const results = new Map<string, number>();
+		for (let next = index + 1; next < messages.length && messages[next]!.role !== "assistant"; next++) {
+			const candidate = messages[next]!;
+			if (candidate.role === "toolResult" && !results.has(candidate.toolCallId)) {
+				results.set(candidate.toolCallId, next);
+			}
+		}
+		for (const call of calls) {
+			const resultIndex = results.get(call.id);
+			ordered.push(resultIndex === undefined ? missingResult(call, message.timestamp) : messages[resultIndex]!);
+		}
+	}
+	return ordered;
+}
+
+function missingResult(call: ToolCall, timestamp: number): ToolResultMessage {
+	return freezeJson({
+		role: "toolResult",
+		toolCallId: call.id,
+		toolName: call.name,
+		content: [{ type: "text", text: MISSING_RESULT_TEXT }],
+		isError: true,
+		details: { reason: "missing_result" },
+		timestamp,
+	});
+}

third_party/pi/packages/durable/src/harness/define.ts

@@ -0,0 +1,44 @@
+import type { JsonValue } from "@earendil-works/chord";
+import type { TSchema } from "@earendil-works/pi-ai";
+import type { AnyTask, Extension, HookRegistration, HooksOf, PromptSection, ToolRegistration, Wrap } from "./types.ts";
+
+/** Identity function that types an extension. */
+export function defineExtension<Tool extends ToolRegistration = ToolRegistration>(
+	extension: Extension<Tool>,
+): Extension<Tool> {
+	return extension;
+}
+
+/** Identity function that types a tool: `args` from `parameters`, details from what it reports. */
+export function defineTool<TParameters extends TSchema, TDetails extends JsonValue = JsonValue>(
+	tool: ToolRegistration<TParameters, TDetails>,
+): ToolRegistration<TParameters, TDetails> {
+	return tool;
+}
+
+/** A prompt section; tagged unless `tag` is false. */
+export function section<Tool extends ToolRegistration = ToolRegistration>(
+	key: string,
+	render: PromptSection<Tool>["render"],
+	options?: { readonly tag?: boolean },
+): PromptSection<Tool> {
+	return options?.tag === undefined ? { key, render } : { key, render, tag: options.tag };
+}
+
+/** Hook handlers for tasks with `task`'s name. */
+export function hook<K extends AnyTask>(task: K, handlers: Partial<HooksOf<K>>): HookRegistration {
+	return { task: task.definition.name, handlers };
+}
+
+/** Wrap the tool named like `tool` wherever the wrapping extension is selected. */
+export function wrapTool<Tool extends ToolRegistration>(tool: Tool, wrapper: (tool: Tool) => Tool): Wrap<Tool> {
+	return { tool: tool.name, wrap: wrapper };
+}
+
+/** Wrap the section `key` wherever the wrapping extension is selected. */
+export function wrapSection<Tool extends ToolRegistration = ToolRegistration>(
+	key: string,
+	wrapper: (section: PromptSection<Tool>) => PromptSection<Tool>,
+): Wrap<Tool> {
+	return { section: key, wrap: wrapper };
+}

third_party/pi/packages/durable/src/harness/events.ts

@@ -0,0 +1,506 @@
+import type { Context, JsonValue } from "@earendil-works/chord";
+import type { Op, Path } from "@earendil-works/chord/delta";
+import type { AssistantMessage, Message, Usage } from "@earendil-works/pi-ai";
+import { CommittedWatch } from "../session/observation.ts";
+import type {
+	CommitChange,
+	CommitPublication,
+	ConversationId,
+	EntryRecord,
+	JsonObject,
+	SubmissionId,
+	SubmissionRecord,
+	TaskId,
+	WatchEnd,
+} from "../types.ts";
+import { AgentDoc } from "./agent.ts";
+import type { InboxItem, InboxState } from "./inbox.ts";
+import type { CompactionFailure, CompactionStatus, LiveState, NestedToolSlot, ToolSlot } from "./live.ts";
+import { NestedCallDoc, type NestedCallState } from "./tool.ts";
+import type { AgentState, CompactionReason, Harness, NestedToolExecutionResult, ToolDiagnostic } from "./types.ts";
+import { UsageDoc, type UsageState } from "./usage.ts";
+import { scanAll } from "./util.ts";
+import { type ConversationView, conversationViews } from "./view.ts";
+
+type Block = AssistantMessage["content"][number];
+type QueuedItem = { id: SubmissionId; mode: InboxItem["mode"] };
+
+/** One change to the in-flight assistant message, relative to that message. */
+export type MessageChange =
+	| { type: "text_start" | "thinking_start" | "toolcall_start"; contentIndex: number; block: Block }
+	| { type: "text_delta" | "thinking_delta"; contentIndex: number; delta: string }
+	| { type: "toolcall_delta"; contentIndex: number; path: readonly (string | number)[]; delta: string }
+	| { type: "block"; contentIndex: number; block: Block }
+	| { type: "message"; message: AssistantMessage };
+
+export type SnapshotEvent = {
+	type: "snapshot";
+	entries: readonly EntryRecord[];
+	run?: { inputs: readonly SubmissionId[] };
+	/** Current generation attempt: its in-flight partial, retry backoff, or deferred poll. */
+	generation?: {
+		attempt: number;
+		message?: AssistantMessage;
+		retry?: { at: number; error: string };
+		deferred?: { pollAt: number };
+	};
+	tools: readonly ToolSlot[];
+	/** `pi.live.nestedTools`: nested calls of running tool calls. */
+	nestedTools: readonly NestedToolSlot[];
+	/** `pi.live.compactions`: live compactions with their attempt and retry backoff. */
+	compactions: readonly CompactionStatus[];
+	/** `pi.live.compactionFailure`: the newest compaction failure since a compaction last produced a summary. */
+	compactionFailure?: CompactionFailure;
+	inbox: readonly QueuedItem[];
+	/** `pi.agent`; `{}` when absent. */
+	agent: AgentState;
+	usage: UsageState;
+};
+
+/** Experimental agent event, shaped like the coding agent's session events (spec §9.4). */
+export type AgentEvent =
+	| SnapshotEvent
+	| { type: "run_start"; inputs: readonly SubmissionId[] }
+	| { type: "run_end"; inputs: readonly SubmissionId[] }
+	| { type: "turn_start" }
+	| { type: "turn_end" }
+	| { type: "message_start"; message: Message }
+	/** `usage` is the partial's current usage, as in the coding agent's JSON mode. */
+	| { type: "message_update"; usage: Usage; changes: readonly MessageChange[] }
+	| { type: "message_end"; entry: EntryRecord }
+	| ({ type: "tool_execution_start"; args: JsonObject } & ToolEventCall)
+	| ({
+			type: "tool_execution_update";
+			/** A front trim and then an append of the retained window, or its replacement. */
+			output?: { trimStart?: number; append?: string } | { set: string };
+			details?: JsonValue;
+			diagnostics?: readonly ToolDiagnostic[];
+	  } & ToolEventCall)
+	/**
+	 * A model-issued call ends with its result `entry`, a nested call with its `result`. Both are absent when the tool
+	 * task faulted or was orphaned, or when the call's slot left `pi.live` unfinished: its run ended, or its parent
+	 * settled first.
+	 */
+	| ({ type: "tool_execution_end"; entry?: EntryRecord; result?: NestedToolExecutionResult } & ToolEventCall)
+	| { type: "inbox_update"; items: readonly QueuedItem[] }
+	| { type: "submission"; record: SubmissionRecord }
+	| { type: "auto_retry_start"; attempt: number; at: number; errorMessage: string }
+	| { type: "auto_retry_end"; attempt: number }
+	| { type: "deferred_poll"; pollAt: number }
+	| { type: "entry_appended"; entry: EntryRecord }
+	| { type: "agent_changed"; agent: AgentState }
+	| { type: "usage_changed"; usage: UsageState }
+	| { type: "task_failed"; taskId: TaskId; kind: string; message: string }
+	| { type: "compaction_start"; taskId: TaskId; reason: CompactionReason; blocking: boolean }
+	/**
+	 * The task's receipt tells whether it produced a summary; the summary entry has its own events. `error` is set when
+	 * the compaction failed, faulted, or was orphaned, as in `pi.live.compactionFailure`.
+	 */
+	| { type: "compaction_end"; taskId: TaskId; reason: CompactionReason; error?: string };
+
+/**
+ * Which call a tool event is about. `toolCallId` is the call's ID in the transcript and in tool events: the provider's
+ * ID for a model-issued call, `<parent call ID>/<key>` for a nested one. Match events by it; do not parse it. `taskId`
+ * is the call's tool task, absent for a call that never got one (not offered, or not started in a sequential round).
+ * A nested call also names the call that made it, by call ID and by tool task.
+ */
+export type ToolEventCall = {
+	toolCallId: string;
+	toolName: string;
+	taskId?: TaskId;
+	parentToolCallId?: string;
+	parentTaskId?: TaskId;
+};
+
+/** Serialized stream of one conversation's event batches, one per commit. */
+export interface AgentEventStream {
+	/** The `snapshot` event at attachment. */
+	readonly snapshot: SnapshotEvent;
+	start(listener: (events: readonly AgentEvent[], context: Context) => Promise<void>): void;
+	stop(): Promise<WatchEnd>;
+	readonly closed: Promise<WatchEnd>;
+}
+
+/** The typed parts of a view the events read. */
+type Parts = {
+	live: LiveState;
+	inbox: InboxState | undefined;
+	agent: AgentState | undefined;
+	usage: UsageState | undefined;
+};
+
+function parts(view: ConversationView): Parts {
+	return {
+		live: (view.docs["pi.live"] ?? {}) as LiveState,
+		inbox: view.docs["pi.inbox"] as InboxState | undefined,
+		agent: view.docs["pi.agent"] as AgentState | undefined,
+		usage: view.docs["pi.usage"] as UsageState | undefined,
+	};
+}
+
+function snapshotOf(view: ConversationView): SnapshotEvent {
+	const { live, inbox, agent, usage } = parts(view);
+	return {
+		type: "snapshot",
+		entries: view.entries,
+		...(live.run === undefined ? {} : { run: { inputs: live.run.inputs } }),
+		...(live.generation === undefined ? {} : { generation: live.generation as SnapshotEvent["generation"] }),
+		tools: live.tools ?? [],
+		nestedTools: live.nestedTools ?? [],
+		compactions: live.compactions ?? [],
+		...(live.compactionFailure === undefined ? {} : { compactionFailure: live.compactionFailure }),
+		inbox: queued(inbox),
+		agent: agent ?? AgentDoc.definition.initial(),
+		usage: usage ?? UsageDoc.definition.initial(),
+	};
+}
+
+function queued(inbox: InboxState | undefined): QueuedItem[] {
+	return (inbox?.items ?? []).map((item) => ({ id: item.id, mode: item.mode }));
+}
+
+/**
+ * Experimental: attach to one conversation's agent events (spec §9.4). The snapshot and the registration for later
+ * commits are captured atomically on the Session line; overflow replaces undelivered batches with one snapshot.
+ */
+export async function watchEvents(
+	harness: Harness,
+	conversationId: ConversationId,
+	context: Context,
+): Promise<AgentEventStream> {
+	const views = conversationViews(harness);
+	let watch!: CommittedWatch<readonly AgentEvent[]>;
+	let snapshot!: SnapshotEvent;
+	await views.attach(
+		conversationId,
+		async (initial, release, storage) => {
+			// Generations whose held outcome already ended their turn, read on the line with the snapshot.
+			const query = { conversationId, kind: "pi.generation", status: "completing" } as const;
+			const completing = await scanAll((cursor) => storage.scanTasks(query, 100, cursor, context));
+			const held = new Set<TaskId>(completing.map((record) => record.id));
+			let current = initial;
+			snapshot = snapshotOf(initial);
+			// Batches are the watch's values; an overflow delivers a snapshot of the newest view instead.
+			watch = new CommittedWatch<readonly AgentEvent[]>(
+				[],
+				release,
+				(error) => views.report(error),
+				() => [snapshotOf(current)],
+			);
+			return {
+				publication: (before, after, ops, publication, commitContext) => {
+					current = after;
+					const events = translate(conversationId, before, after, ops, publication, held);
+					if (events.length > 0) watch.advance(events, [], commitContext);
+				},
+				closeSession: (failure) => watch.closeSession(failure),
+			};
+		},
+		context,
+	);
+	// Like a watch, the acquisition context governs the stream's lifetime.
+	const signal = context.abortSignal;
+	if (signal?.aborted) {
+		watch.cancel();
+		throw signal.reason;
+	}
+	if (signal !== undefined) watch.observeCancellation(signal);
+	return {
+		snapshot,
+		start: (listener) => watch.start((events, _ops, deliveryContext) => listener(events, deliveryContext)),
+		stop: () => watch.stop(),
+		closed: watch.closed,
+	};
+}
+
+type TaskChange = Extract<CommitChange, { type: "task" }>;
+
+/** The tool result for `callId` among `entries`. */
+function resultOf(entries: readonly EntryRecord[], callId: string): EntryRecord | undefined {
+	return entries.find((entry) => {
+		const message = entry.model?.[0];
+		return message?.role === "toolResult" && message.toolCallId === callId;
+	});
+}
+
+/** Every event one publication causes, in the order of spec §9.4. */
+function translate(
+	conversationId: ConversationId,
+	before: ConversationView,
+	after: ConversationView,
+	viewOps: readonly Op[],
+	publication: CommitPublication,
+	held: Set<TaskId>,
+): AgentEvent[] {
+	const entries: EntryRecord[] = [];
+	const tasks = new Map<TaskId, TaskChange["value"]>();
+	const submissions: SubmissionRecord[] = [];
+	// Nested results this commit stored, by nested task ID; read now, since the caller's documents may retire next.
+	const nestedResults = new Map<TaskId, NestedToolExecutionResult>();
+	for (const change of publication.changes) {
+		if (
+			change.type === "document" &&
+			change.record.kind === NestedCallDoc.definition.kind &&
+			change.value !== null &&
+			change.conversationId === conversationId
+		) {
+			const state = change.value as NestedCallState;
+			if (state.taskId !== undefined && state.result !== undefined) {
+				nestedResults.set(state.taskId, state.result as unknown as NestedToolExecutionResult);
+			}
+		}
+		if (change.type === "entry" && change.value.conversationId === conversationId) entries.push(change.value);
+		if (change.type === "task" && change.value.conversationId === conversationId)
+			tasks.set(change.value.id, change.value);
+		if (change.type === "submission" && change.value.conversationId === conversationId) {
+			submissions.push(change.value);
+		}
+	}
+	if (viewOps.length === 0 && entries.length === 0 && tasks.size === 0 && submissions.length === 0) return [];
+	// Entries are appended in ID order; submission records are published in the order the commit first touched them.
+	submissions.sort((a, b) => a.id - b.id);
+	const was = parts(before);
+	const now = parts(after);
+	const events: AgentEvent[] = [];
+
+	// Progress: tool starts, the in-flight message, tool updates, retry and deferred state.
+	const slotsBefore = new Map((was.live.tools ?? []).map((slot) => [slot.callId, slot]));
+	const slots = now.live.tools ?? [];
+	const nestedBefore = new Map((was.live.nestedTools ?? []).map((slot) => [slot.taskId, slot]));
+	const nested = now.live.nestedTools ?? [];
+	for (const slot of [...slots, ...nested]) {
+		const previous = "parentCallId" in slot ? nestedBefore.get(slot.taskId) : slotsBefore.get(slot.callId);
+		if (slot.status !== "running" || previous?.status === "running") continue;
+		// A nested slot carries the arguments the call runs with; a model-issued call's are in its intent checkpoint.
+		const checkpoint = slot.taskId === undefined ? undefined : tasks.get(slot.taskId)?.state.checkpoint;
+		const args =
+			"parentCallId" in slot
+				? slot.arguments
+				: ((checkpoint as { arguments?: JsonObject } | undefined)?.arguments ?? {});
+		events.push({ type: "tool_execution_start", ...callOf(slot), args });
+	}
+	const partialBefore = was.live.generation?.message as AssistantMessage | undefined;
+	const partial = now.live.generation?.message as AssistantMessage | undefined;
+	if (partial !== undefined && partialBefore === undefined) events.push({ type: "message_start", message: partial });
+	else if (partial !== undefined && partial !== partialBefore) {
+		events.push({ type: "message_update", usage: partial.usage, changes: messageChanges(viewOps, partial) });
+	}
+	for (const [index, slot] of slots.entries()) {
+		const previous = slotsBefore.get(slot.callId);
+		if (slot.status !== "running" || previous?.status !== "running") continue;
+		const update = toolUpdate(viewOps, ["tools", index], slot, previous);
+		if (update === undefined) continue;
+		events.push({ type: "tool_execution_update", ...callOf(slot), ...update });
+	}
+	for (const [index, slot] of nested.entries()) {
+		const previous = nestedBefore.get(slot.taskId);
+		if (slot.status !== "running" || previous?.status !== "running") continue;
+		const update = toolUpdate(viewOps, ["nestedTools", index], slot, previous);
+		if (update === undefined) continue;
+		events.push({ type: "tool_execution_update", ...callOf(slot), ...update });
+	}
+	const generation = now.live.generation;
+	const generationBefore = was.live.generation;
+	if (generation?.retry !== undefined && generationBefore?.retry === undefined) {
+		const { at, error } = generation.retry;
+		events.push({ type: "auto_retry_start", attempt: generation.attempt, at, errorMessage: error });
+	}
+	if (generationBefore?.retry !== undefined && generation?.retry === undefined) {
+		events.push({ type: "auto_retry_end", attempt: generationBefore.attempt });
+	}
+	if (generation?.deferred !== undefined && generation.deferred.pollAt !== generationBefore?.deferred?.pollAt) {
+		events.push({ type: "deferred_poll", pollAt: generation.deferred.pollAt });
+	}
+
+	// Tools that end in this commit: a slot that becomes done, one created done (a call not offered), or an unfinished
+	// one that vanishes because its run ended. A done slot that vanishes ended earlier.
+	const toolEnds: Extract<AgentEvent, { type: "tool_execution_end" }>[] = [];
+	const endTool = (slot: ToolSlot, entryId: number | undefined): void => {
+		const entry = entries.find((candidate) => candidate.id === entryId);
+		toolEnds.push({ type: "tool_execution_end", ...callOf(slot), ...(entry === undefined ? {} : { entry }) });
+	};
+	for (const previous of slotsBefore.values()) {
+		if (previous.status === "done") continue;
+		const slot = slots.find((candidate) => candidate.callId === previous.callId);
+		if (slot?.status === "done") endTool(slot, slot.entry);
+		// A slot whose run ended in this commit may have had its result appended with it, as for unstarted calls.
+		else if (slot === undefined) endTool(previous, resultOf(entries, previous.callId)?.id);
+	}
+	for (const slot of slots) {
+		if (slot.status === "done" && !slotsBefore.has(slot.callId)) endTool(slot, slot.entry);
+	}
+	// Nested calls that end in this commit, the same way, children before the calls that made them: the lists hold
+	// parents first, so walk them backwards.
+	const endNested = (slot: NestedToolSlot): void => {
+		const result = nestedResults.get(slot.taskId);
+		events.push({ type: "tool_execution_end", ...callOf(slot), ...(result === undefined ? {} : { result }) });
+	};
+	const nestedNow = new Map(nested.map((slot) => [slot.taskId, slot]));
+	for (const previous of [...nestedBefore.values()].reverse()) {
+		if (previous.status === "done") continue;
+		const slot = nestedNow.get(previous.taskId);
+		if (slot?.status === "done" || slot === undefined) endNested(slot ?? previous);
+	}
+	for (const slot of [...nested].reverse()) {
+		if (slot.status === "done" && !nestedBefore.has(slot.taskId)) endNested(slot);
+	}
+
+	// Entries in append order; a tool's end directly precedes its result's message, as in the coding agent.
+	let assistantAppended = false;
+	for (const entry of entries) {
+		events.push(...toolEnds.filter((end) => end.entry === entry));
+		const message = entry.model?.[0];
+		if (message === undefined) {
+			events.push({ type: "entry_appended", entry });
+			continue;
+		}
+		// A streamed answer already started with its first partial.
+		const streamed = message.role === "assistant" && partialBefore !== undefined && !assistantAppended;
+		if (message.role === "assistant") assistantAppended = true;
+		if (!streamed) events.push({ type: "message_start", message });
+		events.push({ type: "message_end", entry });
+	}
+	// Ends without a result entry: a faulted or orphaned tool, or one whose run ended.
+	events.push(...toolEnds.filter((end) => end.entry === undefined));
+
+	// Compaction ends, task failures, then turn and run ends.
+	const compactionsBefore = was.live.compactions ?? [];
+	const compactions = now.live.compactions ?? [];
+	const failure = now.live.compactionFailure;
+	for (const { taskId, reason } of compactionsBefore) {
+		if (!compactions.some((status) => status.taskId === taskId)) {
+			const error = failure?.taskId === taskId ? failure.error : undefined;
+			events.push({ type: "compaction_end", taskId, reason, ...(error === undefined ? {} : { error }) });
+		}
+	}
+	// A generation's turn ends when its outcome is committed: at a `completing` hold or at terminal, whichever comes
+	// first, so a successor created at the hold starts after it.
+	let turnEnded = false;
+	for (const task of tasks.values()) {
+		const status = task.state.status;
+		if (task.kind === "pi.generation" && status === "completing" && !held.has(task.id)) {
+			held.add(task.id);
+			turnEnded = true;
+		}
+		if (status !== "terminal") continue;
+		if (task.kind === "pi.generation" && !held.delete(task.id)) turnEnded = true;
+		const outcome = task.state.outcome;
+		if (outcome.status === "faulted" || outcome.status === "orphaned") {
+			const message = outcome.status === "faulted" ? outcome.error.message : outcome.reason;
+			events.push({ type: "task_failed", taskId: task.id, kind: task.kind, message });
+		}
+	}
+	if (turnEnded) events.push({ type: "turn_end" });
+	const run = now.live.run;
+	const runBefore = was.live.run;
+	const runChanged = run?.inputs[0] !== runBefore?.inputs[0];
+	if (runBefore !== undefined && runChanged) events.push({ type: "run_end", inputs: runBefore.inputs });
+
+	// Submissions, document state, then what began.
+	for (const record of submissions) events.push({ type: "submission", record });
+	if (now.inbox !== was.inbox) events.push({ type: "inbox_update", items: queued(now.inbox) });
+	// A retired document reads as its initial value, as in a snapshot.
+	if (now.agent !== was.agent) {
+		events.push({ type: "agent_changed", agent: now.agent ?? AgentDoc.definition.initial() });
+	}
+	if (now.usage !== was.usage)
+		events.push({ type: "usage_changed", usage: now.usage ?? UsageDoc.definition.initial() });
+	for (const { taskId, reason, blocking } of compactions) {
+		if (!compactionsBefore.some((status) => status.taskId === taskId)) {
+			events.push({ type: "compaction_start", taskId, reason, blocking });
+		}
+	}
+	if (run !== undefined && runChanged) events.push({ type: "run_start", inputs: run.inputs });
+	if (run !== undefined && run.taskId !== runBefore?.taskId && tasks.get(run.taskId)?.kind === "pi.generation") {
+		events.push({ type: "turn_start" });
+	}
+	return events;
+}
+
+const PARTIAL_PATH = ["docs", "pi.live", "generation", "message"];
+
+/** Translate the view operations on the in-flight message into message changes (spec §9.4). */
+function messageChanges(viewOps: readonly Op[], message: AssistantMessage): MessageChange[] {
+	const changes: MessageChange[] = [];
+	// A block sent whole already holds every later change to it in this batch.
+	const whole = new Set<number>();
+	for (const op of viewOps) {
+		// View operations never replace the root.
+		const path = op[1] as Path;
+		if (!startsWith(path, PARTIAL_PATH)) {
+			// The whole message or generation was replaced.
+			if (startsWith(PARTIAL_PATH, path)) return [{ type: "message", message }];
+			continue;
+		}
+		const rest = path.slice(PARTIAL_PATH.length);
+		if (rest[0] === "usage") continue;
+		if (rest[0] !== "content") return [{ type: "message", message }];
+		if (rest.length === 1) {
+			if (op[0] !== "p" || op[3] !== 0) return [{ type: "message", message }];
+			for (const [offset, block] of (op[4] as unknown as Block[]).entries()) {
+				const type =
+					block.type === "text" ? "text_start" : block.type === "thinking" ? "thinking_start" : "toolcall_start";
+				changes.push({ type, contentIndex: op[2] + offset, block });
+			}
+			continue;
+		}
+		const contentIndex = rest[1] as number;
+		const field = rest[2];
+		if (whole.has(contentIndex)) continue;
+		if (op[0] === "a" && rest.length === 3 && (field === "text" || field === "thinking")) {
+			changes.push({ type: field === "text" ? "text_delta" : "thinking_delta", contentIndex, delta: op[2] });
+		} else if (op[0] === "a" && field === "arguments") {
+			changes.push({ type: "toolcall_delta", contentIndex, path: rest.slice(3), delta: op[2] });
+		} else {
+			whole.add(contentIndex);
+			changes.push({ type: "block", contentIndex, block: message.content[contentIndex]! });
+		}
+	}
+	return changes;
+}
+
+/** The identifying fields of a slot's tool events. */
+function callOf(slot: ToolSlot | NestedToolSlot): ToolEventCall {
+	return {
+		toolCallId: slot.callId,
+		toolName: slot.name,
+		...(slot.taskId === undefined ? {} : { taskId: slot.taskId }),
+		...("parentCallId" in slot ? { parentToolCallId: slot.parentCallId, parentTaskId: slot.parentTaskId } : {}),
+	};
+}
+
+/** Output, details, and diagnostics changes of a running slot at `at` in `pi.live`, from the view operations on it. */
+function toolUpdate(
+	viewOps: readonly Op[],
+	at: readonly ["tools" | "nestedTools", number],
+	slot: ToolSlot | NestedToolSlot,
+	previous: ToolSlot | NestedToolSlot,
+): Omit<Extract<AgentEvent, { type: "tool_execution_update" }>, "type" | keyof ToolEventCall> | undefined {
+	const outputPath = ["docs", "pi.live", ...at, "output"];
+	let trimStart = 0;
+	let append = "";
+	let set = false;
+	for (const op of viewOps) {
+		if (!startsWith(op[1] as Path, outputPath)) continue;
+		if (op[0] === "t") trimStart += op[2];
+		else if (op[0] === "a") append += op[2];
+		else set = true;
+	}
+	let output: { trimStart?: number; append?: string } | { set: string } | undefined;
+	if (set || (slot.output !== previous.output && trimStart === 0 && append === ""))
+		output = { set: slot.output ?? "" };
+	else if (trimStart > 0 || append !== "") {
+		output = { ...(trimStart > 0 ? { trimStart } : {}), ...(append === "" ? {} : { append }) };
+	}
+	// A safe replay clears a running slot's progress: removed details send `null`, removed diagnostics `[]`.
+	const update = {
+		...(output === undefined ? {} : { output }),
+		...(slot.details === previous.details ? {} : { details: slot.details ?? null }),
+		...(slot.diagnostics === previous.diagnostics ? {} : { diagnostics: slot.diagnostics ?? [] }),
+	};
+	return Object.keys(update).length === 0 ? undefined : update;
+}
+
+function startsWith(path: Path, prefix: readonly (string | number)[]): boolean {
+	return prefix.length <= path.length && prefix.every((segment, index) => path[index] === segment);
+}

third_party/pi/packages/durable/src/harness/generation.ts

@@ -0,0 +1,715 @@
+import { type Context, copyJson, type Draft, type JsonValue } from "@earendil-works/chord";
+import type {
+	Api,
+	AssistantMessage,
+	DeferredHandle,
+	Message,
+	Model,
+	ModelThinkingLevel,
+	SimpleStreamOptions,
+	ToolCall,
+} from "@earendil-works/pi-ai";
+import { isContextOverflow } from "@earendil-works/pi-ai/utils/overflow";
+import { isRetryableAssistantError, retryDelayMs } from "@earendil-works/pi-ai/utils/retry";
+import { getCurrentTools } from "@earendil-works/pi-ai/utils/transcript";
+import { AssistantEntry, ResetEntry, SystemEntry, UserEntry } from "../entries.ts";
+import type { ExecutionEnv } from "../env/index.ts";
+import { defineTask } from "../tasks.ts";
+import type {
+	ConversationId,
+	EntryId,
+	NextTaskState,
+	SubmissionId,
+	TaskId,
+	TaskOutcome,
+	TaskRuntime,
+	Tx,
+	TypedEntry,
+} from "../types.ts";
+import { addTools } from "./agent.ts";
+import { createCompaction, estimateContext, selectCut } from "./compaction.ts";
+import { applyBoundary, prepareBoundary } from "./inbox.ts";
+import { assignJson } from "./json.ts";
+import { endRun, LiveDoc, type LiveState, type ToolSlot } from "./live.ts";
+import { planSystemEntries, renderSections, replaySections } from "./prompt.ts";
+import { ensureProviderSessionId } from "./provider.ts";
+import { appendToolResult, harnessError, ToolTask, type ToolTaskResult } from "./tool.ts";
+import type {
+	CompactionPolicy,
+	CompactionResult,
+	ContextView,
+	ConversationStreamOptions,
+	GenerationHooks,
+	ModelRef,
+	PromptInput,
+	ToolControl,
+	UserInput,
+} from "./types.ts";
+import { recordUsage } from "./usage.ts";
+
+export type GenerationInput = Record<string, never>;
+
+export type GenerationCheckpoint =
+	| {
+			phase: "prepare";
+			attempt: number;
+			/** The blocking compaction this generation waited for; it starts no other compaction (spec §8.3). */
+			compacted?: TaskId<CompactionResult>;
+			/** Error text of the overflow that started `compacted`; checked once when `prepare` resumes. */
+			overflow?: string;
+			/** Set when the blocking threshold started `compacted`; its outcome is checked once when `prepare` resumes. */
+			threshold?: true;
+	  }
+	| {
+			phase: "request";
+			attempt: number;
+			compacted?: TaskId<CompactionResult>;
+			model: ModelRef;
+			thinkingLevel: ModelThinkingLevel;
+			/** The settings' request options when preparation committed; a resend after recovery uses them unchanged. */
+			streamOptions: ConversationStreamOptions;
+			/** Newest entry included in the request. */
+			cutoff: EntryId;
+	  }
+	| { phase: "retry"; attempt: number; compacted?: TaskId<CompactionResult>; until: number }
+	| {
+			phase: "poll";
+			attempt: number;
+			compacted?: TaskId<CompactionResult>;
+			model: ModelRef;
+			cutoff: EntryId;
+			handle: DeferredHandle;
+			pollAt: number;
+	  }
+	| {
+			/** Waiting on the round's tool tasks, which the generation owns (spec §8.5). */
+			phase: "tools";
+			/** The tool-calling answer. */
+			assistant: EntryId;
+			/** Tool tasks created so far, in call order; grows by one per started call of a sequential round. */
+			tools: TaskId<ToolTaskResult>[];
+			/** Calls of a sequential round not started yet, in call order. */
+			pending: string[];
+	  };
+
+export type GenerationResult = { entryId: EntryId };
+
+type Runtime = TaskRuntime<GenerationInput, GenerationCheckpoint, GenerationResult, GenerationHooks>;
+type Next = NextTaskState<GenerationCheckpoint, GenerationResult>;
+
+/** What classification needs from the request that produced a message. */
+type Request = {
+	readonly attempt: number;
+	readonly compacted: TaskId<CompactionResult> | undefined;
+	readonly model: ModelRef;
+	readonly cutoff: EntryId;
+	/** Committed model context through `cutoff`, when the phase already derived it. */
+	readonly messages?: readonly Message[];
+	/** Set when the message came from polling, so a still deferred result polls strictly later. */
+	readonly pollAt?: number;
+};
+
+const DEFAULT_POLL_AFTER_MS = 5000;
+
+/**
+ * Built-in generation task: prepares the positional system prompt and tool loadout, requests or polls the model,
+ * retries, and classifies the response. The run's inputs live in `pi.live.run`.
+ */
+export const GenerationTask = defineTask<GenerationInput, GenerationCheckpoint, GenerationResult, GenerationHooks>({
+	name: "pi.generation",
+	version: 1,
+	initial: () => ({ phase: "prepare", attempt: 1 }),
+	phases: {
+		/**
+		 * Render the system prompt and tool loadout and append the positional `pi.system` entries they need, then move to
+		 * `request`. The agent and settings resolved here are fixed for this request. Only the Harness writes model
+		 * context to a busy conversation, so the context read here is still current at the commit; display-only entries
+		 * may land meanwhile.
+		 */
+		prepare: async (task, runtime, context) => {
+			const { conversationId } = runtime;
+			const agent = await runtime.agent(context);
+			const settings = runtime.settings;
+			const { model, thinkingLevel } = agent;
+			const resolved = model === undefined ? undefined : runtime.models.getModel(model.provider, model.modelId);
+			if (model === undefined || resolved === undefined) return failNoModel(runtime, model, context);
+			const { attempt, compacted, overflow, threshold: blocked } = task.state.checkpoint;
+			if (compacted !== undefined && overflow !== undefined) {
+				const [outcome] = await runtime.outcomes([compacted], context);
+				if (outcome?.status !== "completed" || outcome.result.entryId === undefined) {
+					return failModelError(runtime, overflow, context);
+				}
+			}
+			// A threshold compaction that found nothing to compact or was declined leaves a request that may still fit. One
+			// that failed left a context it judged too large: sending it would at best leave no room for the answer.
+			if (compacted !== undefined && blocked === true) {
+				const [outcome] = await runtime.outcomes([compacted], context);
+				if (outcome !== undefined && outcome.status !== "completed") {
+					return failModelError(runtime, compactionFailure(outcome), context);
+				}
+			}
+			const view = await runtime.context(conversationId, context);
+			const shown = replaySections(view.messages);
+			const report = (error: unknown) => runtime.report(error);
+			let env: ExecutionEnv | undefined;
+			try {
+				env = await runtime.env(context);
+			} catch (error) {
+				if (context.abortSignal?.aborted) throw error;
+				report(error);
+			}
+			const input: PromptInput = { conversationId, agent, env, shown: Object.fromEntries(shown), read: runtime };
+			const desired = await renderSections(agent.sections, input, shown, report, context);
+			const entries = planSystemEntries(view, desired, agent.tools, runtime.now());
+			const threshold =
+				compacted === undefined
+					? thresholdCompaction(view, entries, resolved.contextWindow, settings.compaction)
+					: undefined;
+			if (threshold === "blocking") {
+				// Compact first and prepare again; the transcript is unchanged until the compaction appends.
+				await runtime.commit(async (tx): Promise<Next> => {
+					const child = await createCompaction(tx, conversationId, { reason: "threshold" }, runtime.taskId);
+					const checkpoint = { phase: "prepare", attempt, compacted: child, threshold: true } as const;
+					return { status: "waiting", checkpoint, on: [child], policy: "allSettled" };
+				}, context);
+				return;
+			}
+			await runtime.commit(async (tx) => {
+				let cutoff = (await tx.scanEntries({ conversationId }, 1)).items[0]?.id;
+				for (const entry of entries) cutoff = (await tx.appendEntry(SystemEntry, conversationId, entry)).id;
+				if (cutoff === undefined) throw new Error(`Conversation ${conversationId} has no entries to send`);
+				// Checked in this commit, so a compaction admitted during preparation counts.
+				if (threshold === "background" && (await tx.doc(LiveDoc, conversationId)).compactions === undefined) {
+					await createCompaction(tx, conversationId, { reason: "threshold" });
+				}
+				const request = {
+					attempt,
+					...(compacted === undefined ? {} : { compacted }),
+					model,
+					thinkingLevel,
+					streamOptions: settings.stream,
+					cutoff,
+				};
+				return { status: "running", checkpoint: { phase: "request", ...request } };
+			}, context);
+		},
+		request: async (task, runtime, context) => {
+			const { attempt, compacted, model: ref, thinkingLevel, streamOptions, cutoff } = task.state.checkpoint;
+			const conversationId = runtime.conversationId;
+			await runtime.commit(async (tx) => {
+				const live = await tx.doc(LiveDoc, conversationId);
+				await convertPartial(tx, live, conversationId);
+				live.generation = { attempt };
+				return undefined;
+			}, context);
+			const model = runtime.models.getModel(ref.provider, ref.modelId);
+			if (model === undefined) return failNoModel(runtime, ref, context);
+			const view = await runtime.context(conversationId, context, { at: cutoff });
+			let messages = view.messages;
+			await runtime.hooks.each("beforeRequest", async (hook) => {
+				const replaced = await hook({ messages }, runtime, context);
+				if (replaced !== undefined) messages = replaced.messages;
+			});
+			const options: SimpleStreamOptions = {
+				...streamOptions,
+				signal: runtime.signal,
+				sessionId: await ensureProviderSessionId(runtime, context),
+				...(thinkingLevel === "off" ? {} : { reasoning: thinkingLevel }),
+			};
+			const message = await streamResponse(runtime, model, messages, options, attempt, context);
+			const request = { attempt, compacted, model: ref, cutoff, messages: view.messages };
+			await classify(runtime, request, message, context);
+		},
+		retry: async (task, runtime, context) => {
+			const { attempt, compacted, until } = task.state.checkpoint;
+			await runtime.sleep(until, context);
+			await runtime.commit(async (tx) => {
+				(await tx.doc(LiveDoc, runtime.conversationId)).generation = { attempt: attempt + 1 };
+				const checkpoint: GenerationCheckpoint = {
+					phase: "prepare",
+					attempt: attempt + 1,
+					...(compacted === undefined ? {} : { compacted }),
+				};
+				return { status: "running", checkpoint };
+			}, context);
+		},
+		poll: async (task, runtime, context) => {
+			const { attempt, compacted, model: ref, cutoff, handle, pollAt } = task.state.checkpoint;
+			const model = runtime.models.getModel(ref.provider, ref.modelId);
+			if (model === undefined) return failNoModel(runtime, ref, context);
+			await runtime.sleep(pollAt, context);
+			const message = await runtime.models.fetchDeferred(model, handle, { signal: runtime.signal });
+			const request = { attempt, compacted, model: ref, cutoff, pollAt };
+			await classify(runtime, request, message, context);
+		},
+		tools: async (task, runtime, context) => {
+			const { assistant, tools, pending } = task.state.checkpoint;
+			const [next, ...rest] = pending;
+			if (next === undefined) return finishToolRound(runtime, assistant, tools, context);
+			// Sequential round: start the next call and wait for it.
+			await runtime.commit(async (tx): Promise<Next> => {
+				const live = await tx.doc(LiveDoc, runtime.conversationId);
+				const taskId = await createToolTask(tx, runtime, assistant, next);
+				const slot = live.tools?.find((slot) => slot.callId === next && slot.taskId === undefined);
+				if (slot !== undefined) slot.taskId = taskId;
+				const checkpoint: GenerationCheckpoint = {
+					phase: "tools",
+					assistant,
+					tools: [...tools, taskId],
+					pending: rest,
+				};
+				return { status: "waiting", checkpoint, on: [taskId], policy: "allSettled" };
+			}, context);
+		},
+	},
+	abort: async (task, runtime, context) => {
+		const checkpoint = task.state.checkpoint;
+		if (checkpoint.phase === "poll") {
+			const model = runtime.models.getModel(checkpoint.model.provider, checkpoint.model.modelId);
+			if (model !== undefined) {
+				try {
+					await runtime.models.cancelDeferred(model, checkpoint.handle, { signal: runtime.signal });
+				} catch (error) {
+					runtime.report(error);
+				}
+			}
+		}
+		const conversationId = runtime.conversationId;
+		// Runs after the round's tool tasks are terminal; calls never started get `aborted` results (spec §8.5).
+		const unstarted =
+			checkpoint.phase === "tools"
+				? await readCalls(runtime, checkpoint.assistant, checkpoint.pending, context)
+				: [];
+		await runtime.commit(async (tx) => {
+			const live = await tx.doc(LiveDoc, conversationId);
+			await convertPartial(tx, live, conversationId);
+			for (const call of unstarted) {
+				const result = harnessError("aborted", `Tool ${call.name} was aborted`);
+				await appendToolResult(tx, conversationId, call, result, { timestamp: runtime.now() });
+			}
+			endRun(tx, live, runtime.taskId, { status: "unanswered", reason: "aborted" });
+			return { status: "terminal", outcome: { status: "aborted" } };
+		}, context);
+	},
+});
+
+/** The calls `callIds` of the assistant entry, in the given order. */
+async function readCalls(
+	runtime: Runtime,
+	assistant: EntryId,
+	callIds: readonly string[],
+	context: Context,
+): Promise<ToolCall[]> {
+	const message = (await runtime.entry(AssistantEntry, assistant, context))?.model?.[0];
+	const calls = message?.role === "assistant" ? message.content.filter((content) => content.type === "toolCall") : [];
+	return callIds.flatMap((id) => calls.filter((call) => call.id === id).slice(0, 1));
+}
+
+/** A tool task for call `callId`, owned by the generation. */
+function createToolTask(tx: Tx, runtime: Runtime, assistant: EntryId, callId: string): Promise<TaskId<ToolTaskResult>> {
+	return tx.createTask(
+		ToolTask,
+		{ kind: "model", assistant, callId },
+		{ ownership: { kind: "task", taskId: runtime.taskId } },
+	);
+}
+
+/**
+ * Which threshold compaction preparation starts before its request (spec §8.3): `blocking` above
+ * `contextWindow - reserveTokens`, `background` above the background threshold, and only when range selection finds a
+ * cut. The caller starts a background one only while no compaction is listed.
+ */
+function thresholdCompaction(
+	view: ContextView,
+	planned: readonly { readonly model?: readonly Message[] }[],
+	contextWindow: number,
+	policy: CompactionPolicy,
+): "blocking" | "background" | undefined {
+	if (!policy.enabled || contextWindow <= 0) return undefined;
+	const tokens = estimateContext(
+		view,
+		planned.flatMap((entry) => entry.model ?? []),
+	);
+	const blocking = contextWindow - policy.reserveTokens;
+	const background = blocking - policy.backgroundTokens;
+	const over =
+		tokens > blocking ? "blocking" : policy.backgroundTokens > 0 && tokens > background ? "background" : undefined;
+	if (over === undefined || selectCut(view, policy.keepRecentTokens) === undefined) return undefined;
+	return over;
+}
+
+/** Error text for a generation whose blocking compaction ended without completing. */
+function compactionFailure(outcome: Exclude<TaskOutcome<CompactionResult>, { status: "completed" }>): string {
+	switch (outcome.status) {
+		case "failed":
+		case "faulted":
+			return `Compaction failed: ${outcome.error.message}`;
+		case "orphaned":
+			return `Compaction failed: ${outcome.reason}`;
+		case "aborted":
+			return "Compaction was aborted";
+	}
+}
+
+/** Settle the run's inputs `unanswered` with `model_error` and fail with `text`. */
+async function failModelError(runtime: Runtime, text: string, context: Context): Promise<void> {
+	await runtime.commit(async (tx) => {
+		const live = await tx.doc(LiveDoc, runtime.conversationId);
+		endRun(tx, live, runtime.taskId, { status: "unanswered", reason: "model_error", detail: text });
+		return {
+			status: "terminal",
+			outcome: { status: "failed", error: { message: text, detail: { reason: "model_error" } } },
+		};
+	}, context);
+}
+
+/** Settle the run's inputs `unanswered` with `no_model` and fail. */
+async function failNoModel(runtime: Runtime, ref: ModelRef | undefined, context: Context): Promise<void> {
+	const message =
+		ref === undefined ? "No model is configured" : `Model ${ref.provider}/${ref.modelId} is not available`;
+	await runtime.commit(async (tx) => {
+		const live = await tx.doc(LiveDoc, runtime.conversationId);
+		endRun(tx, live, runtime.taskId, { status: "unanswered", reason: "no_model" });
+		return { status: "terminal", outcome: { status: "failed", error: { message, detail: { reason: "no_model" } } } };
+	}, context);
+}
+
+/**
+ * Append a committed partial left by an interrupted, aborted, faulted, or orphaned attempt as an aborted assistant
+ * entry; the caller replaces or removes `generation`.
+ */
+export async function convertPartial(tx: Tx, live: Draft<LiveState>, conversationId: ConversationId): Promise<void> {
+	const partial = live.generation?.message;
+	if (partial === undefined) return;
+	const message = copyJson(partial) as unknown as AssistantMessage;
+	await appendAssistant(tx, conversationId, { ...message, stopReason: "aborted" });
+}
+
+/**
+ * Stream one request and return the terminal message. Partials commit as trailing writes at most every
+ * `progress.partialIntervalMs` (default 100 ms) with one commit in flight; `finally` stops the throttle and awaits that
+ * commit, so no stale partial lands after the outcome.
+ */
+async function streamResponse(
+	runtime: Runtime,
+	model: Model<Api>,
+	messages: readonly Message[],
+	options: SimpleStreamOptions,
+	attempt: number,
+	context: Context,
+): Promise<AssistantMessage> {
+	const interval = runtime.settings.progress.partialIntervalMs;
+	let pending: AssistantMessage | undefined;
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	let inFlight: Promise<void> | undefined;
+	let stopped = false;
+	const flush = (): void => {
+		timer = undefined;
+		const partial = pending;
+		pending = undefined;
+		if (partial === undefined || stopped) return;
+		inFlight = (async () => {
+			// Copy synchronously: the provider keeps mutating its partial.
+			const message = copyJson(partial, { omitUndefinedProperties: true });
+			await runtime.commit(async (tx) => {
+				const live = await tx.doc(LiveDoc, runtime.conversationId);
+				live.generation ??= { attempt };
+				assignJson(live.generation as Draft<Record<string, JsonValue>>, "message", message);
+				return undefined;
+			}, context);
+		})()
+			.catch((error: unknown) => {
+				// Rejections after an abort mark or close are expected; the committed state stays consistent.
+				if (!runtime.signal.aborted) runtime.report(error);
+			})
+			.finally(() => {
+				inFlight = undefined;
+				if (pending !== undefined && !stopped) timer = setTimeout(flush, interval);
+			});
+	};
+	try {
+		const events = runtime.models.streamSimple(model, { messages: [...messages] }, options);
+		for await (const event of events) {
+			// A partial without content, such as pi-ai's opening `start` event, shows nothing; a deferred response
+			// never gets past it, so it never leaves a partial.
+			if (event.type === "done" || event.type === "error" || event.partial.content.length === 0) continue;
+			pending = event.partial;
+			if (timer === undefined && inFlight === undefined) timer = setTimeout(flush, interval);
+		}
+		return await events.result();
+	} finally {
+		stopped = true;
+		clearTimeout(timer);
+		await inFlight;
+	}
+}
+
+/** Classify a terminal provider message in one commit that also clears the partial. */
+async function classify(
+	runtime: Runtime,
+	request: Request,
+	message: AssistantMessage,
+	context: Context,
+): Promise<void> {
+	// An abort mark or close: the abort invocation or the reopened run handles the committed state.
+	runtime.signal.throwIfAborted();
+	const conversationId = runtime.conversationId;
+	const { attempt, compacted, model: ref, cutoff } = request;
+	if (message.stopReason === "deferred" && message.deferred !== undefined) {
+		const handle = message.deferred;
+		const pollAt = Math.max(
+			runtime.now() + (handle.pollAfterMs ?? DEFAULT_POLL_AFTER_MS),
+			request.pollAt === undefined ? Number.NEGATIVE_INFINITY : request.pollAt + 1,
+		);
+		await runtime.commit(async (tx) => {
+			(await tx.doc(LiveDoc, conversationId)).generation = { attempt, deferred: { pollAt } };
+			const checkpoint = {
+				phase: "poll",
+				attempt,
+				...(compacted === undefined ? {} : { compacted }),
+				model: ref,
+				cutoff,
+				handle,
+				pollAt,
+			} as const;
+			return { status: "running", checkpoint };
+		}, context);
+		return;
+	}
+	await runtime.hooks.each("afterResponse", (hook) => hook(message, runtime, context));
+	const calls = message.content.filter((content): content is ToolCall => content.type === "toolCall");
+	if (message.stopReason === "toolUse" && calls.length > 0) {
+		return startToolRound(runtime, request, message, calls, context);
+	}
+	if (message.stopReason === "stop" || message.stopReason === "length" || message.stopReason === "toolUse") {
+		return answer(runtime, message, context);
+	}
+	// The retry and compaction policies govern the next attempt, so they are read now rather than pinned at preparation.
+	const settings = runtime.settings;
+	const overflow = message.stopReason === "error" && isContextOverflow(message);
+	if (overflow && compacted === undefined && settings.compaction.enabled) {
+		const policy = settings.compaction;
+		const view = await runtime.context(conversationId, context, { at: cutoff });
+		if (selectCut(view, policy.keepRecentTokens) !== undefined) {
+			const text = message.errorMessage ?? "Context overflow";
+			await runtime.commit(async (tx): Promise<Next> => {
+				const live = await tx.doc(LiveDoc, conversationId);
+				await appendAssistant(tx, conversationId, message);
+				delete live.generation;
+				const child = await createCompaction(tx, conversationId, { reason: "overflow" }, runtime.taskId);
+				const checkpoint = { phase: "prepare", attempt, compacted: child, overflow: text } as const;
+				return { status: "waiting", checkpoint, on: [child], policy: "allSettled" };
+			}, context);
+			return;
+		}
+	}
+	const policy = settings.retry;
+	// An overflow is never retried: only a compaction can make the next request fit.
+	const retry =
+		message.stopReason === "error" &&
+		!overflow &&
+		isRetryableAssistantError(message) &&
+		policy.enabled &&
+		attempt <= policy.maxRetries;
+	const until = retry ? runtime.now() + retryDelayMs(policy, attempt) : 0;
+	await runtime.commit(async (tx): Promise<Next> => {
+		const live = await tx.doc(LiveDoc, conversationId);
+		await appendAssistant(tx, conversationId, message);
+		if (retry) {
+			live.generation = { attempt, retry: { at: until, error: message.errorMessage ?? "" } };
+			const checkpoint = {
+				phase: "retry",
+				attempt,
+				...(compacted === undefined ? {} : { compacted }),
+				until,
+			} as const;
+			return { status: "running", checkpoint };
+		}
+		const text = message.errorMessage ?? `Model response ended with stop reason ${message.stopReason}`;
+		endRun(tx, live, runtime.taskId, { status: "unanswered", reason: "model_error", detail: text });
+		return {
+			status: "terminal",
+			outcome: { status: "failed", error: { message: text, detail: { reason: "model_error" } } },
+		};
+	}, context);
+}
+
+/**
+ * A final answer; the final boundary places queued items (spec §6). The first `onYield` continuation appends a user
+ * message and hands the run to a successor generation, but only when the boundary selected no user item and no reset.
+ * Otherwise the run's inputs settle `done`, and selected user items start the next run.
+ */
+async function answer(runtime: Runtime, message: AssistantMessage, context: Context): Promise<void> {
+	let continuation: UserInput | undefined;
+	await runtime.hooks.each("onYield", async (hook) => {
+		if (continuation !== undefined) return;
+		continuation = (await hook(message, runtime, context))?.continue;
+	});
+	const conversationId = runtime.conversationId;
+	await runtime.commit(async (tx): Promise<Next> => {
+		// Queue modes are read on the Session line, when the boundary is decided.
+		const boundary = await prepareBoundary(tx, conversationId, runtime.settings);
+		const live = await tx.doc(LiveDoc, conversationId);
+		const entry = await appendAssistant(tx, conversationId, message);
+		const result: Next = { status: "terminal", outcome: { status: "completed", result: { entryId: entry.id } } };
+		const { users, reset } = await applyBoundary(tx, boundary, "final", runtime.now());
+		if (continuation !== undefined && users.length === 0 && !reset) {
+			const user = { role: "user", content: continuation, timestamp: runtime.now() } as const;
+			await tx.appendEntry(UserEntry, conversationId, { model: [user] });
+			handOver(live, runtime.taskId, await createGeneration(tx, conversationId));
+			delete live.generation;
+			return result;
+		}
+		endRun(tx, live, runtime.taskId, { status: "done", answer: entry.id });
+		if (users.length > 0) await startRun(tx, conversationId, live, users);
+		return result;
+	}, context);
+}
+
+/**
+ * Append the tool-calling answer and start its tool round in one commit (spec §8.3). A call to a tool the request did
+ * not offer gets its `tool_unavailable` result here; every other call gets a tool task owned by the generation, only the
+ * first one now when the round is sequential. The generation then waits for them in its `tools` phase, keeping the run.
+ */
+async function startToolRound(
+	runtime: Runtime,
+	request: Request,
+	message: AssistantMessage,
+	calls: readonly ToolCall[],
+	context: Context,
+): Promise<void> {
+	const conversationId = runtime.conversationId;
+	const messages =
+		request.messages ?? (await runtime.context(conversationId, context, { at: request.cutoff })).messages;
+	const offered = new Set(getCurrentTools(messages).map((tool) => tool.name));
+	// Read as the round starts; a tool is resolved as its tool task resolves it.
+	const tools = (await runtime.agent(context)).tools;
+	const sequential =
+		runtime.settings.toolExecution === "sequential" ||
+		calls.some(
+			(call) =>
+				offered.has(call.name) && tools.find((tool) => tool.name === call.name)?.executionMode === "sequential",
+		);
+	await runtime.commit(async (tx): Promise<Next> => {
+		const live = await tx.doc(LiveDoc, conversationId);
+		const entry = await appendAssistant(tx, conversationId, message);
+		const slots: ToolSlot[] = [];
+		const tools: TaskId<ToolTaskResult>[] = [];
+		const pending: string[] = [];
+		for (const call of calls) {
+			if (!offered.has(call.name)) {
+				const unavailable = harnessError("tool_unavailable", `Tool ${call.name} is not available`);
+				const result = await appendToolResult(tx, conversationId, call, unavailable, { timestamp: runtime.now() });
+				slots.push({ callId: call.id, name: call.name, status: "done", entry: result.id });
+				continue;
+			}
+			if (sequential && tools.length > 0) {
+				pending.push(call.id);
+				slots.push({ callId: call.id, name: call.name, status: "pending" });
+				continue;
+			}
+			const taskId = await createToolTask(tx, runtime, entry.id, call.id);
+			tools.push(taskId);
+			slots.push({ callId: call.id, name: call.name, taskId, status: "pending" });
+		}
+		delete live.generation;
+		live.tools = slots;
+		const checkpoint = { phase: "tools", assistant: entry.id, tools, pending } as const;
+		return { status: "waiting", checkpoint, on: tools, policy: "allSettled" };
+	}, context);
+}
+
+/**
+ * The round's tools are terminal: apply their controls and either end the run at the final boundary (`terminate`,
+ * `handoff`, or a queued reset) or hand it to the next generation at the `postTools` boundary (spec §8.5).
+ */
+async function finishToolRound(
+	runtime: Runtime,
+	assistant: EntryId,
+	tools: readonly TaskId<ToolTaskResult>[],
+	context: Context,
+): Promise<void> {
+	const conversationId = runtime.conversationId;
+	const controls = new Map<TaskId, ToolControl | undefined>();
+	const outcomes = await runtime.outcomes(tools, context);
+	tools.forEach((id, index) => {
+		const outcome = outcomes[index]!;
+		// Version 1 results have no kind; every result a generation's round reads is a model-issued call's.
+		controls.set(
+			id,
+			outcome.status === "completed" && outcome.result.kind !== "nested" ? outcome.result.control : undefined,
+		);
+	});
+	const slots = (await runtime.snapshot(LiveDoc, conversationId, context))?.tools ?? [];
+	const results = slots.flatMap((slot) => (slot.entry === undefined ? [] : [slot.entry]));
+	await runtime.hooks.each("afterTools", (hook) => hook(assistant, results, runtime, context));
+	// Every call of the round, including those answered without a task, must ask to terminate.
+	const terminate =
+		slots.length > 0 &&
+		slots.every((slot) => slot.taskId !== undefined && controls.get(slot.taskId)?.terminate === true);
+	const added = [...controls.values()].flatMap((control) => control?.addTools ?? []);
+	// The last handoff in call order wins.
+	const handoff = [...controls.values()].findLast((control) => control?.handoff !== undefined)?.handoff;
+	await runtime.commit(async (tx): Promise<Next> => {
+		const boundary = await prepareBoundary(tx, conversationId, runtime.settings);
+		if (added.length > 0) await addTools(tx, conversationId, added);
+		const live = await tx.doc(LiveDoc, conversationId);
+		const now = runtime.now();
+		if (terminate || handoff !== undefined) {
+			if (handoff !== undefined) {
+				const message = { role: "user", content: handoff, timestamp: now } as const;
+				const entry = await tx.appendEntry(ResetEntry, conversationId, { head: "self", model: [message] });
+				boundary.head = entry.id;
+			}
+			const { users } = await applyBoundary(tx, boundary, "final", now);
+			endRun(tx, live, runtime.taskId, { status: "done", answer: assistant });
+			if (users.length > 0) await startRun(tx, conversationId, live, users);
+		} else {
+			const { users, reset } = await applyBoundary(tx, boundary, "postTools", now);
+			if (reset) {
+				// The queued reset cut the run's context before an answer.
+				endRun(tx, live, runtime.taskId, { status: "unanswered", reason: "reset" });
+				if (users.length > 0) await startRun(tx, conversationId, live, users);
+			} else {
+				delete live.tools;
+				delete live.nestedTools;
+				if (live.run?.taskId === runtime.taskId) live.run.inputs.push(...users);
+				handOver(live, runtime.taskId, await createGeneration(tx, conversationId));
+			}
+		}
+		return { status: "terminal", outcome: { status: "completed", result: { entryId: assistant } } };
+	}, context);
+}
+
+/**
+ * Append a provider result and add its usage to `pi.usage` in the same commit.
+ * REMINDER: every built-in writer of assistant entries goes through here, so the usage ledger stays complete.
+ */
+async function appendAssistant(
+	tx: Tx,
+	conversationId: ConversationId,
+	message: AssistantMessage,
+): Promise<TypedEntry<never>> {
+	await recordUsage(tx, conversationId, "models", `${message.provider}/${message.model}`, message.usage);
+	return tx.appendEntry(AssistantEntry, conversationId, { model: [message] });
+}
+
+/** Start a run for `inputs`, placed input submissions: a new generation takes `pi.live.run`. */
+export async function startRun(
+	tx: Tx,
+	conversationId: ConversationId,
+	live: Draft<LiveState>,
+	inputs: SubmissionId[],
+): Promise<void> {
+	live.run = { taskId: await createGeneration(tx, conversationId), inputs };
+}
+
+/** A generation owned by its conversation. */
+function createGeneration(tx: Tx, conversationId: ConversationId): Promise<TaskId<GenerationResult>> {
+	return tx.createTask(GenerationTask, {}, { ownership: { kind: "conversation" }, conversationId });
+}
+
+/** Hand run control from `from` to `to`; the run's inputs move with it. */
+export function handOver(live: Draft<LiveState>, from: TaskId, to: TaskId): void {
+	if (live.run?.taskId === from) live.run.taskId = to;
+}

third_party/pi/packages/durable/src/harness/harness.ts

@@ -0,0 +1,470 @@
+import type { AttachedReplicatedState, Context, JsonValue } from "@earendil-works/chord";
+import { withAbortSignal, withoutAbortSignal } from "@earendil-works/chord/context";
+import { ResetEntry } from "../entries.ts";
+import type { ExecutionEnv } from "../env/index.ts";
+import { contained, SessionImpl } from "../session/session.ts";
+import type { Transaction } from "../session/transaction.ts";
+import type {
+	ConversationId,
+	ConversationOwnership,
+	ConversationRecord,
+	Cursor,
+	EntryId,
+	EntryQuery,
+	EntryRecord,
+	Page,
+	Storage,
+	SubmissionId,
+	TaskId,
+	TaskRecord,
+	Tx,
+} from "../types.ts";
+import { ROOT_CONVERSATION_ID } from "../types.ts";
+import { AgentDoc, configure, createAgent, resolveAgent, resolveSettings } from "./agent.ts";
+import { createCompaction } from "./compaction.ts";
+import { readContext } from "./context.ts";
+import { InboxDoc, withdrawQueuedInputs } from "./inbox.ts";
+import { LiveDoc, settleSchedulerOutcome } from "./live.ts";
+import { ProviderDoc } from "./provider.ts";
+import { BUILTIN_TASKS } from "./registry.ts";
+import { type InvocationBinding, TaskScheduler } from "./scheduler.ts";
+import { Submissions } from "./submissions.ts";
+import { type TaskGraph, TaskGraphView, type TaskGraphWatch } from "./task-graph.ts";
+import type {
+	Agent,
+	AgentChange,
+	CompactionResult,
+	ContextView,
+	Conversation,
+	ConversationAbortOptions,
+	ConversationCreateOptions,
+	ConversationHandle,
+	ConversationInit,
+	ConversationWatch,
+	HarnessInspection,
+	HarnessOptions,
+	Harness as HarnessType,
+	RegistrySnapshot,
+	SettledTask,
+	Submission,
+	SubmissionDraft,
+	ToolRegistration,
+} from "./types.ts";
+import { addUsageState, UsageDoc, type UsageState } from "./usage.ts";
+import { scanAll } from "./util.ts";
+import { type ConversationView, ConversationViews } from "./view.ts";
+
+const SCAN_PAGE_SIZE = 256;
+
+type CreateTarget =
+	| { readonly kind: "root" }
+	| { readonly kind: "independent"; readonly ownership: ConversationOwnership }
+	| {
+			readonly kind: "fork";
+			readonly parentId: ConversationId;
+			readonly at: EntryId;
+			readonly ownership: ConversationOwnership;
+	  };
+
+/** What the conveniences apply in the creating commit, after the creation hook. */
+type CreateOptions = { readonly agent?: AgentChange; readonly init?: ConversationInit };
+
+/** Harness-private services used by Conversation handles. */
+type ConversationHost<Tool extends ToolRegistration> = {
+	readonly harness: HarnessImpl<Tool>;
+	readonly storage: Storage;
+	readonly tasks: TaskScheduler;
+	readonly submissions: Submissions;
+	readonly views: ConversationViews;
+	readonly now: () => number;
+	create(target: CreateTarget, options: CreateOptions, context: Context): Promise<Conversation>;
+};
+
+/** Key of the Harness's internal index-size getter, for tests. */
+export const SCHEDULER_INDEX_SIZES = Symbol.for("pi-durable.schedulerIndexSizes");
+
+class ConversationImpl<Tool extends ToolRegistration> implements Conversation {
+	readonly id: ConversationId;
+	readonly #host: ConversationHost<Tool>;
+
+	constructor(id: ConversationId, host: ConversationHost<Tool>) {
+		this.id = id;
+		this.#host = host;
+	}
+
+	agent(context: Context): Promise<Agent> {
+		return this.#host.harness.resolveAgent(this.id, undefined, context);
+	}
+
+	configure(change: AgentChange, context: Context): Promise<void> {
+		return this.#host.harness.commitWith((tx) => configure(tx, this.id, change), context);
+	}
+
+	submit(submission: SubmissionDraft, context: Context): Promise<Submission> {
+		return this.#host.submissions.submit(this.id, submission, context);
+	}
+
+	compact(instructions: string | undefined, context: Context): Promise<TaskId<CompactionResult>> {
+		this.#host.tasks.resume();
+		const input = { reason: "manual", ...(instructions === undefined ? {} : { instructions }) } as const;
+		return this.#host.harness.commitWith((tx) => createCompaction(tx, this.id, input), context);
+	}
+
+	async reset(handoff: string | undefined, context: Context): Promise<void> {
+		const model =
+			handoff === undefined
+				? {}
+				: { model: [{ role: "user", content: handoff, timestamp: this.#host.now() } as const] };
+		const entry = { kind: ResetEntry.kind, head: "self", ...model } as const;
+		await this.#host.submissions.submit(this.id, { type: "write", entry }, context);
+	}
+
+	commit<T>(change: (tx: Tx) => T | Promise<T>, context: Context): Promise<T> {
+		return this.#host.harness.commitWith(change, context, { conversationId: this.id });
+	}
+
+	context(context: Context, options?: { readonly at?: EntryId }): Promise<ContextView> {
+		return readContext(this.#host.harness, this.#host.storage, this.id, context, options?.at);
+	}
+
+	entries(
+		query: Omit<EntryQuery, "conversationId">,
+		limit: number,
+		cursor: Cursor | undefined,
+		context: Context,
+	): Promise<Page<EntryRecord, Cursor>> {
+		const bounded: EntryQuery = {
+			conversationId: this.id,
+			...(query.minEntryId === undefined ? {} : { minEntryId: query.minEntryId }),
+			...(query.maxEntryId === undefined ? {} : { maxEntryId: query.maxEntryId }),
+			...(query.order === undefined ? {} : { order: query.order }),
+		};
+		return this.#host.harness.readOnLine(() => this.#host.storage.scanEntries(bounded, limit, cursor, context));
+	}
+
+	fork(at: EntryId, options: ConversationCreateOptions, context: Context): Promise<Conversation> {
+		return this.#host.create({ kind: "fork", parentId: this.id, at, ownership: options.ownership }, options, context);
+	}
+
+	abort(context: Context, options?: ConversationAbortOptions): Promise<void> {
+		this.#host.tasks.resume();
+		return this.#host.tasks.abortConversation(this.id, options?.background === true, context);
+	}
+
+	waitForIdle(context: Context): Promise<void> {
+		this.#host.tasks.resume();
+		return this.#host.tasks.waitForIdle(this.id, context);
+	}
+
+	viewState(context: Context): Promise<AttachedReplicatedState<ConversationView>> {
+		return this.#host.views.state(this.id, context);
+	}
+
+	watch(context: Context): Promise<ConversationWatch> {
+		return this.#host.views.watch(this.id, context);
+	}
+}
+
+/** Session kernel extended with conversation handles and a registry. */
+class HarnessImpl<Tool extends ToolRegistration> extends SessionImpl implements HarnessType {
+	readonly #options: HarnessOptions<Tool>;
+	readonly #report: (error: unknown) => void;
+	readonly #host: ConversationHost<Tool>;
+	readonly #tasks: TaskScheduler;
+	readonly #submissions: Submissions;
+	readonly #taskGraph: TaskGraphView;
+
+	constructor(rawStorage: Storage, options: HarnessOptions<Tool>, context: Context) {
+		const report = safeReport(options.onReport);
+		const now = safeNow(options.now, report);
+		// Read at each use, as all settings are, but only this field: it is read on every commit.
+		super(rawStorage, now, () => options.settings?.documentRetentionMs);
+		// Every component reads and writes through the Session's failure guard: `rawStorage` serves only `super()`.
+		const storage = this.storage;
+		this.#options = options;
+		this.#report = report;
+		const settings = () => resolveSettings(options.settings);
+		this.#tasks = new TaskScheduler({
+			session: this,
+			storage,
+			registry: options.registry,
+			models: options.models,
+			agent: (id, snapshot, callContext) => this.resolveAgent(id, snapshot as RegistrySnapshot<Tool>, callContext),
+			settings,
+			env: (id, callContext) => this.buildEnv(id, callContext),
+			now,
+			report: this.#report,
+			settleOutcome: settleSchedulerOutcome,
+			withdrawInputs: withdrawQueuedInputs,
+			conversation: async (id, binding, callContext) => {
+				const record = await this.readOnLine(() => storage.conversation(id, callContext));
+				return record === undefined ? undefined : boundConversation(id, binding, this.#submissions, this.#tasks);
+			},
+			context: withoutAbortSignal(context),
+		});
+		this.#submissions = new Submissions(this, storage, now, settings, () => this.#tasks.resume());
+		this.#taskGraph = new TaskGraphView(this, storage);
+		this.#host = {
+			harness: this,
+			storage,
+			tasks: this.#tasks,
+			submissions: this.#submissions,
+			views: new ConversationViews(this, storage),
+			now,
+			create: (target, createOptions, context) => this.#create(target, createOptions, context),
+		};
+	}
+
+	/** Resolve a conversation's committed `pi.agent` against `snapshot`, or the current one, and the current settings. */
+	async resolveAgent(
+		id: ConversationId,
+		snapshot: RegistrySnapshot<Tool> | undefined,
+		context: Context,
+	): Promise<Agent<Tool>> {
+		const registry = snapshot ?? this.#options.registry.snapshot();
+		const state = await this.snapshot(AgentDoc, id, context);
+		return resolveAgent(state, registry, resolveSettings(this.#options.settings), this.#report);
+	}
+
+	/** Build a conversation's environment from its current `cwd`; `undefined` without an `env` option. */
+	async buildEnv(id: ConversationId, context: Context): Promise<ExecutionEnv | undefined> {
+		const build = this.#options.env;
+		if (build === undefined) return undefined;
+		const cwd = (await this.snapshot(AgentDoc, id, context))?.cwd;
+		return build({ conversationId: id, ...(cwd === undefined ? {} : { cwd }), read: this }, context);
+	}
+
+	/** Reconcile surviving `running` tasks to `pending`; part of open. */
+	openTasks(context: Context): Promise<void> {
+		return this.#tasks.open(context);
+	}
+
+	resume(): void {
+		this.assertUsable();
+		this.#tasks.resume();
+	}
+
+	getTask<R>(id: TaskId<R>, context: Context): Promise<TaskRecord<JsonValue, JsonValue, R> | undefined> {
+		return this.readOnLine(() => this.storage.task(id, context)) as Promise<
+			TaskRecord<JsonValue, JsonValue, R> | undefined
+		>;
+	}
+
+	/** Internal, for tests: sizes of the scheduler's ownership indexes. */
+	get [SCHEDULER_INDEX_SIZES](): TaskScheduler["indexSizes"] {
+		return this.#tasks.indexSizes;
+	}
+
+	inspect(context: Context): Promise<HarnessInspection> {
+		return this.readOnLine(async () => {
+			const { scheduling, tasks } = await this.#tasks.inspect(this.#options.registry.snapshot());
+			const scan = (status: "queued" | "placed") =>
+				scanAll((cursor) => this.storage.scanSubmissions({ status }, SCAN_PAGE_SIZE, cursor, context));
+			const submissions = [...(await scan("queued")), ...(await scan("placed"))].sort((a, b) => a.id - b.id);
+			return { scheduling, tasks, submissions };
+		});
+	}
+
+	submission(id: SubmissionId, context: Context): Promise<Submission | undefined> {
+		return this.#submissions.get(id, context);
+	}
+
+	abortSubmission(
+		id: SubmissionId,
+		context: Context,
+		conversationId?: ConversationId,
+	): Promise<"aborted" | "already_placed" | "settled" | "not_found"> {
+		return this.#submissions.abort(id, context, conversationId);
+	}
+
+	abortTask(id: TaskId, context: Context): Promise<"marked" | "terminal"> {
+		return this.#tasks.abort(id, context);
+	}
+
+	waitForTask<R>(id: TaskId<R>, context: Context): Promise<SettledTask<R>> {
+		this.#tasks.resume();
+		return this.#tasks.waitForTask(id, context) as Promise<SettledTask<R>>;
+	}
+
+	waitForIdle(context: Context): Promise<void> {
+		this.#tasks.resume();
+		return this.#tasks.waitForIdle(undefined, context);
+	}
+
+	/** Sum every conversation's committed `pi.usage`. Each document is read at its own point; totals only grow. */
+	async usage(context: Context): Promise<UsageState> {
+		const conversations = await this.readOnLine(() =>
+			scanAll((cursor) => this.storage.scanConversations({}, SCAN_PAGE_SIZE, cursor, context)),
+		);
+		const total = UsageDoc.definition.initial();
+		for (const { id } of conversations) {
+			const state = await this.snapshot(UsageDoc, id, context);
+			if (state !== undefined) addUsageState(total, state);
+		}
+		return total;
+	}
+
+	taskGraph(context: Context): Promise<AttachedReplicatedState<TaskGraph>> {
+		return this.#taskGraph.state(context);
+	}
+
+	watchTaskGraph(context: Context): Promise<TaskGraphWatch> {
+		return this.#taskGraph.watch(context);
+	}
+
+	root(
+		context: Context,
+		options?: { readonly agent?: AgentChange; readonly init?: ConversationInit },
+	): Promise<Conversation> {
+		return this.#create({ kind: "root" }, options ?? {}, context);
+	}
+
+	async conversation(id: ConversationId, context: Context): Promise<Conversation | undefined> {
+		this.assertUsable();
+		const record = await this.readOnLine(() => this.storage.conversation(id, context));
+		return record === undefined ? undefined : new ConversationImpl(record.id, this.#host);
+	}
+
+	createConversation(options: ConversationCreateOptions, context: Context): Promise<Conversation> {
+		return this.#create({ kind: "independent", ownership: options.ownership }, options, context);
+	}
+
+	override report(error: unknown): void {
+		this.#report(error);
+	}
+
+	/** Signal and join task invocations after the close listeners and before Storage closes; writes no task outcome. */
+	protected override beforeClose(): Promise<void> {
+		return this.#tasks.join();
+	}
+
+	async #create(target: CreateTarget, options: CreateOptions, context: Context): Promise<Conversation> {
+		this.assertUsable();
+		const id = await this.commitWith(async (tx) => {
+			if (target.kind === "root" && (await tx.conversation(ROOT_CONVERSATION_ID)) !== undefined) {
+				return ROOT_CONVERSATION_ID;
+			}
+			const record =
+				target.kind === "root"
+					? await tx.createRootConversation()
+					: target.kind === "fork"
+						? await tx.forkConversation(target.parentId, target.at, { ownership: target.ownership })
+						: await tx.createConversation({ ownership: target.ownership });
+			if (options.agent !== undefined) await configure(tx, record.id, options.agent);
+			if (options.init !== undefined) await options.init(tx, record.id);
+			return record.id;
+		}, context);
+		return new ConversationImpl(id, this.#host);
+	}
+
+	/**
+	 * The built-in creation hook, in every commit that creates or forks a conversation: empty `pi.live`, `pi.inbox`, and
+	 * `pi.usage`, a fresh `pi.provider`, the conversation's `pi.agent` (see `createAgent()`), then
+	 * `HarnessOptions.conversationCreated`.
+	 */
+	protected override async conversationCreated(tx: Transaction, record: ConversationRecord): Promise<void> {
+		await tx.doc(LiveDoc, record.id);
+		await tx.doc(InboxDoc, record.id);
+		await tx.doc(UsageDoc, record.id);
+		await tx.doc(ProviderDoc, record.id);
+		await createAgent(tx, record);
+		await this.#options.conversationCreated?.(tx, record);
+	}
+}
+
+/** `onReport` that cannot fail: it is the last place an error goes, so its own throw or rejection is dropped. */
+function safeReport(onReport: ((error: unknown) => void) | undefined): (error: unknown) => void {
+	return (error) =>
+		contained(
+			() => onReport?.(error),
+			() => {},
+		);
+}
+
+/**
+ * The host clock, falling back to `Date.now` when it throws: a broken clock must not stop commits. The first throw is
+ * reported; the clock is read in every task change, so later ones are not.
+ */
+export function safeNow(now: (() => number) | undefined, report: (error: unknown) => void): () => number {
+	if (now === undefined) return Date.now;
+	let reported = false;
+	return () => {
+		try {
+			return now();
+		} catch (error) {
+			// Set first: a report handler that reads the clock again must not report again.
+			if (!reported) {
+				reported = true;
+				report(error);
+			}
+			return Date.now();
+		}
+	};
+}
+
+/**
+ * Invocation-bound handle for tasks and tools. Every operation, and every operation of a submission it returns, first
+ * checks the invocation and runs under its signal, so it rejects once the invocation ends; admitted work stays durable.
+ */
+function boundConversation(
+	id: ConversationId,
+	binding: InvocationBinding,
+	submissions: Submissions,
+	tasks: TaskScheduler,
+): ConversationHandle {
+	const bind = (context: Context): Context => withAbortSignal(binding.signal, context);
+	const bound = <T>(operation: (context: Context) => Promise<T>) => {
+		return async (context: Context): Promise<T> => {
+			binding.check();
+			return operation(bind(context));
+		};
+	};
+	return {
+		id,
+		submit: async (draft, context) => {
+			binding.check();
+			const submission = await submissions.submit(id, draft, bind(context));
+			return {
+				id: submission.id,
+				status: bound((callContext) => submission.status(callContext)),
+				wait: bound((callContext) => submission.wait(callContext)),
+				abort: bound((callContext) => submission.abort(callContext)),
+			};
+		},
+		abort: async (context, options) => {
+			binding.check();
+			return tasks.abortConversation(id, options?.background === true, bind(context));
+		},
+		waitForIdle: bound((callContext) => tasks.waitForIdle(id, callContext)),
+	};
+}
+
+/** Durable agent harness over one Session. */
+export type Harness = HarnessType;
+
+export const Harness = {
+	/** Open a Harness over storage. The registry may keep changing while the Harness runs. */
+	async open<Tool extends ToolRegistration>(
+		storage: Storage,
+		options: HarnessOptions<Tool>,
+		context: Context,
+	): Promise<Harness> {
+		context.abortSignal?.throwIfAborted();
+		const snapshot = options.registry.snapshot();
+		const missing = BUILTIN_TASKS.filter((task) => snapshot.task(task.definition.name) === undefined);
+		if (missing.length > 0) {
+			const names = missing.map((task) => task.definition.name).join(", ");
+			throw new Error(`Registry lacks built-in tasks ${names}; create it with createRegistry()`);
+		}
+		const harness = new HarnessImpl(storage, options, context);
+		try {
+			await harness.openTasks(context);
+		} catch (error) {
+			// The caller's context may be what failed open: close without it, and rethrow the open error.
+			// A failing Storage close fails the Session, which reports it.
+			await harness.close(withoutAbortSignal(context)).catch(() => {});
+			throw error;
+		}
+		return harness;
+	},
+};

third_party/pi/packages/durable/src/harness/inbox.ts

@@ -0,0 +1,168 @@
+import { copyJson, type Draft, type JsonRepresentation } from "@earendil-works/chord";
+import { defineDoc } from "../documents.ts";
+import { UserEntry } from "../entries.ts";
+import type { ConversationId, EntryDraft, EntryId, EntryRecord, JsonObject, SubmissionId, Tx } from "../types.ts";
+import type { InputEntry, QueueMode, Settings, UserInput } from "./types.ts";
+
+/** A queued submission: user input for a run, with the entry it is placed as when not `pi.user`, or an entry write. */
+export type InboxItem =
+	| {
+			id: SubmissionId;
+			mode: "steer" | "followUp";
+			content: JsonRepresentation<UserInput>;
+			entry?: JsonRepresentation<InputEntry>;
+	  }
+	/** `entry` is an `EntryDraft`, stored as plain JSON. */
+	| { id: SubmissionId; mode: "write"; entry: JsonObject };
+
+/** Built-in queue of one conversation's submissions waiting for a boundary, in ID order. */
+export type InboxState = { items: InboxItem[] };
+
+export const InboxDoc = defineDoc<InboxState>({
+	kind: "pi.inbox",
+	version: 1,
+	scope: "conversation",
+	history: "latest",
+	fork: "initial",
+	initial: () => ({ items: [] }),
+	checkpointWhen: (value) => value.items.length === 0,
+});
+
+/** The settings a boundary reads, on the Session line. */
+export type QueueModes = Pick<Settings, "steeringMode" | "followUpMode">;
+
+/** What a boundary reads before the commit's first table write, and the newest head it has seen so far. */
+export type Boundary = {
+	readonly conversationId: ConversationId;
+	readonly inbox: Draft<InboxState>;
+	readonly steeringMode: QueueMode;
+	readonly followUpMode: QueueMode;
+	/** Start of the active range, the newest head marker's `head`; advanced by heads written in this commit. */
+	head: EntryId | undefined;
+};
+
+/** Selected user items, in ID order, and whether a `head: "self"` write (a reset) was placed. */
+export type BoundaryResult = { readonly users: SubmissionId[]; readonly reset: boolean };
+
+/**
+ * Read what a boundary needs. Table reads must precede the commit's first table write, so callers prepare the
+ * boundary at the start of their commit.
+ */
+export async function prepareBoundary(tx: Tx, conversationId: ConversationId, modes: QueueModes): Promise<Boundary> {
+	const head = (await tx.latestHeadMarker(conversationId))?.head;
+	const inbox = await tx.doc(InboxDoc, conversationId);
+	return {
+		conversationId,
+		inbox,
+		steeringMode: modes.steeringMode,
+		followUpMode: modes.followUpMode,
+		head,
+	};
+}
+
+/**
+ * Place the queued items a boundary selects (spec §6): every write, the first or all steers, and at `final` the first
+ * or all follow-ups. A selected reset turns a `postTools` boundary into `final`. Writes are placed first and user
+ * items after them, each in ID order, so user items queued before a reset run in the new context. A write whose head
+ * targets an entry before the active range, including a range started earlier in this commit, is stale. Selected and
+ * stale items are removed positionally.
+ */
+export async function applyBoundary(
+	tx: Tx,
+	boundary: Boundary,
+	at: "postTools" | "final",
+	now: number,
+): Promise<BoundaryResult> {
+	const { conversationId, inbox } = boundary;
+	const items = inbox.items;
+	const reset = hasQueuedReset(items);
+	const final = at === "final" || reset;
+	const pick = (mode: "steer" | "followUp", queueMode: QueueMode): number[] => {
+		const indexes = items.flatMap((item, index) => (item.mode === mode ? [index] : []));
+		return queueMode === "all" ? indexes : indexes.slice(0, 1);
+	};
+	const writes = items.flatMap((item, index) => (item.mode === "write" ? [index] : []));
+	const users = [...pick("steer", boundary.steeringMode), ...(final ? pick("followUp", boundary.followUpMode) : [])];
+	users.sort((a, b) => a - b);
+
+	// `appendEntry()` copies the drafts' values; the items are removed only afterwards.
+	for (const index of writes) {
+		const item = items[index] as Draft<Extract<InboxItem, { mode: "write" }>>;
+		const draft = item.entry as unknown as EntryDraft;
+		if (isStale(boundary, draft)) {
+			tx.settleSubmission(item.id, { status: "unanswered", reason: "stale" });
+			continue;
+		}
+		const entry = await tx.appendEntry(conversationId, draft);
+		if (draft.head !== undefined) boundary.head = draft.head === "self" ? entry.id : draft.head;
+		tx.placeSubmission(item.id, entry.id);
+	}
+	const placed: SubmissionId[] = [];
+	for (const index of users) {
+		const item = items[index] as Draft<Extract<InboxItem, { mode: "steer" | "followUp" }>>;
+		const entry = await appendInput(tx, conversationId, item.content as UserInput, item.entry as InputEntry, now);
+		tx.placeSubmission(item.id, entry.id);
+		placed.push(item.id);
+	}
+	const removed = [...writes, ...users].sort((a, b) => b - a);
+	for (const index of removed) items.splice(index, 1);
+	return { users: placed, reset };
+}
+
+/** Append the entry an input is placed as: `entry`'s kind and data, or `pi.user`, with the user message of `content`. */
+export function appendInput(
+	tx: Tx,
+	conversationId: ConversationId,
+	content: UserInput,
+	entry: InputEntry | undefined,
+	now: number,
+): Promise<EntryRecord> {
+	const message = { role: "user", content, timestamp: now } as const;
+	if (entry === undefined) return tx.appendEntry(UserEntry, conversationId, { model: [message] });
+	return tx.appendEntry(conversationId, { ...pickInputEntry(entry), model: [message] });
+}
+
+/** The kind and data of an input's entry as strict JSON, without other fields an untyped caller might pass. */
+export function pickInputEntry({ kind, data }: InputEntry): JsonRepresentation<InputEntry> {
+	const picked = data === undefined ? { kind } : { kind, data };
+	return copyJson(picked, { omitUndefinedProperties: true }) as JsonRepresentation<InputEntry>;
+}
+
+/**
+ * Whether a write is display-only: it has no `model`, `head`, or `edits` (present counts, even if empty), so it changes
+ * nothing the model sees and needs no boundary.
+ */
+export function isDisplayOnly(entry: EntryDraft): boolean {
+	return entry.model === undefined && entry.head === undefined && entry.edits === undefined;
+}
+
+/** Whether a reset (`head: "self"` write) is queued; entries appended before it leave the active range when it lands. */
+export function hasQueuedReset(items: readonly InboxItem[]): boolean {
+	return items.some((item) => item.mode === "write" && item.entry.head === "self");
+}
+
+/** Whether a head write targets an entry before the active range, so placing it would bring back cut history. */
+export function isStale(boundary: Boundary, entry: EntryDraft): boolean {
+	return typeof entry.head === "number" && boundary.head !== undefined && entry.head < boundary.head;
+}
+
+/** Remove a withdrawn submission's item; the caller settles the submission. */
+export async function removeInboxItem(tx: Tx, conversationId: ConversationId, id: SubmissionId): Promise<void> {
+	const items = (await tx.doc(InboxDoc, conversationId)).items;
+	const index = items.findIndex((item) => item.id === id);
+	if (index >= 0) items.splice(index, 1);
+}
+
+/**
+ * Withdraw every queued input of a conversation, as `Conversation.abort()` and abort cascades do: each settles
+ * `unanswered` with `aborted` and leaves the inbox; queued writes stay for later placement.
+ */
+export async function withdrawQueuedInputs(tx: Tx, conversationId: ConversationId): Promise<void> {
+	const items = (await tx.doc(InboxDoc, conversationId)).items;
+	for (let index = items.length - 1; index >= 0; index--) {
+		const item = items[index]!;
+		if (item.mode === "write") continue;
+		tx.settleSubmission(item.id, { status: "unanswered", reason: "aborted" });
+		items.splice(index, 1);
+	}
+}

third_party/pi/packages/durable/src/harness/json.ts

@@ -0,0 +1,31 @@
+import type { JsonValue } from "@earendil-works/chord";
+
+type JsonContainer = Record<string, JsonValue> | JsonValue[];
+
+/**
+ * Assign `value` at `target[key]` leaf by leaf. Chord records a container assignment as one full set and only emits an
+ * append when a string leaf is reassigned with a longer string, so writing the partial whole would store and publish the
+ * complete message on every flush.
+ */
+export function assignJson(target: JsonContainer, key: string | number, value: JsonValue): void {
+	const slots = target as Record<string | number, JsonValue>;
+	const current = slots[key];
+	if (isRecord(current) && isRecord(value)) {
+		for (const name of Object.keys(current)) if (!Object.hasOwn(value, name)) delete current[name];
+		for (const [name, child] of Object.entries(value)) assignJson(current, name, child);
+		return;
+	}
+	if (Array.isArray(current) && Array.isArray(value) && current.length <= value.length) {
+		const items = current as JsonValue[];
+		for (let index = 0; index < value.length; index++) {
+			if (index < items.length) assignJson(items, index, value[index]!);
+			else items.push(value[index]!);
+		}
+		return;
+	}
+	if (current !== value) slots[key] = value;
+}
+
+function isRecord(value: JsonValue | undefined): value is Record<string, JsonValue> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}

third_party/pi/packages/durable/src/harness/live.ts

@@ -0,0 +1,268 @@
+import type { Draft, JsonRepresentation, JsonValue } from "@earendil-works/chord";
+import type { AssistantMessage, Usage } from "@earendil-works/pi-ai";
+import { defineDoc } from "../documents.ts";
+import type { Transaction } from "../session/transaction.ts";
+import type { EntryId, JsonObject, SubmissionId, SubmissionSettlement, TaskId, TaskRecord, Tx } from "../types.ts";
+import { convertPartial } from "./generation.ts";
+import type { SchedulerOutcome } from "./scheduler.ts";
+import type { CompactionReason, CompactionResult, ToolDiagnostic } from "./types.ts";
+
+/** What a running tool call publishes, shared by model-issued and nested calls. */
+type SlotProgress = {
+	callId: string;
+	name: string;
+	status: "pending" | "running" | "done";
+	/** Retained running output and what the bounds dropped. */
+	output?: string;
+	droppedBytes?: number;
+	droppedLines?: number;
+	/** Last `details()` value. */
+	details?: JsonValue;
+	/** Diagnostics recorded through `api.diagnostic()`. */
+	diagnostics?: ToolDiagnostic[];
+};
+
+/** Presentation of one tool call of the current round. */
+export type ToolSlot = SlotProgress & {
+	/**
+	 * Absent for a call not started yet (sequential round) and for a call its request did not offer, which starts `done`
+	 * with the `entry` generation wrote.
+	 */
+	taskId?: TaskId;
+	/** Result entry once done; absent when the tool task faulted or was orphaned. */
+	entry?: EntryId;
+};
+
+/**
+ * Presentation of one nested call: a call a running tool made through `executeTool()`. `callId` is
+ * `<parentCallId>/<key>`, the call's ID in tool events and unique because keys contain no `/`. The Harness relates
+ * slots by task ID: `taskId` is the call's tool task, `parentTaskId` the calling one, model-issued or nested.
+ */
+export type NestedToolSlot = SlotProgress & {
+	parentCallId: string;
+	parentTaskId: TaskId;
+	taskId: TaskId;
+	/**
+	 * What the call was made with, replaced by the arguments it runs with, after repair, `beforeTool`, and coercion,
+	 * once it starts executing. A model-issued call's arguments are in its assistant entry instead.
+	 */
+	arguments: JsonObject;
+	/** Once done: how the call ended, for a status line. The result itself is in the caller's `NestedCallDoc`. */
+	summary?: NestedToolSummary;
+};
+
+/** How a nested call ended: an error flag, its execution time, its spend, and up to 500 characters of error text. */
+export type NestedToolSummary = {
+	isError: boolean;
+	durationMs?: number;
+	usage?: JsonRepresentation<Usage>;
+	error?: string;
+};
+
+/** Presentation of one live compaction task (spec §8.7). */
+export type CompactionStatus = {
+	taskId: TaskId<CompactionResult>;
+	reason: CompactionReason;
+	/** Whether a generation waits for it: a compaction the generation owns. */
+	blocking: boolean;
+	attempt: number;
+	/** Durable backoff before the next summarization attempt. */
+	retry?: { at: number; error: string };
+};
+
+/** The newest compaction that failed since a compaction last produced a summary (spec §8.7). */
+export type CompactionFailure = {
+	taskId: TaskId<CompactionResult>;
+	reason: CompactionReason;
+	/** The failed or faulted outcome's error message, or why the scheduler orphaned the task. */
+	error: string;
+};
+
+/** Built-in live conversation state: run control and presentation of the current generation and tool round. */
+export type LiveState = {
+	/** Run control: the task that settles the run's inputs, and those inputs; present exactly while busy. */
+	run?: { taskId: TaskId; inputs: SubmissionId[] };
+	/** Presentation of the current generation attempt. */
+	generation?: {
+		attempt: number;
+		/** Committed throttled partial of the in-flight response. */
+		message?: JsonRepresentation<AssistantMessage>;
+		/** Durable backoff before the next attempt. */
+		retry?: { at: number; error: string };
+		/** Provider-side deferred response being polled. */
+		deferred?: { pollAt: number };
+	};
+	/** The current tool round in call order, from the tool-calling answer until the generation's `tools` phase ends it. */
+	tools?: ToolSlot[];
+	/**
+	 * Nested calls of running tool calls, in creation order, so a parent precedes its children and task IDs ascend. A
+	 * call's nested slots are removed when the call settles.
+	 */
+	nestedTools?: NestedToolSlot[];
+	/** Live compaction tasks in task ID order; absent when none. */
+	compactions?: CompactionStatus[];
+	/**
+	 * The newest compaction failure, so a client can show it after the task ended. A compaction that produces a summary
+	 * removes it; one that is aborted or finds nothing to compact leaves it.
+	 */
+	compactionFailure?: CompactionFailure;
+};
+
+export const LiveDoc = defineDoc<LiveState>({
+	kind: "pi.live",
+	version: 1,
+	scope: "conversation",
+	history: "latest",
+	fork: "initial",
+	initial: () => ({}),
+	// REMINDER: a complete base whenever nothing runs (spec §8.2): no generation and no running tool slot. That holds
+	// while idle, in the commit handing a generation over to its tool round, and between tools, so the delta chain
+	// spans at most one generation or the overlapping execution of one round's tools. A slot holds output only while
+	// running, so every base is small; a nested call's result lives in its caller's task documents, never here. Do not
+	// add a delta-count bound; the tool output benchmark checks this rule.
+	checkpointWhen: (value) =>
+		value.generation === undefined &&
+		!(value.tools ?? []).some((slot) => slot.status === "running") &&
+		!(value.nestedTools ?? []).some((slot) => slot.status === "running"),
+});
+
+/** Built-in task kinds that can own `pi.live.run`. */
+const RUN_TASK_KINDS: ReadonlySet<string> = new Set(["pi.generation"]);
+const TOOL_TASK_KIND = "pi.tool";
+const COMPACTION_TASK_KIND = "pi.compaction";
+
+/**
+ * End the run owned by `taskId`: settle each of its inputs and remove `run`. Always removes `generation`, `tools`, and
+ * `nestedTools`, whose presentation belongs to the ending run.
+ */
+export function endRun(tx: Tx, live: Draft<LiveState>, taskId: TaskId, settlement: SubmissionSettlement): void {
+	if (live.run?.taskId === taskId) {
+		for (const id of live.run.inputs) tx.settleSubmission(id, settlement);
+		delete live.run;
+	}
+	delete live.generation;
+	delete live.tools;
+	delete live.nestedTools;
+}
+
+/** Add the status of a compaction task created in this commit; statuses stay in task ID order. */
+export function addCompactionStatus(live: Draft<LiveState>, status: CompactionStatus): void {
+	live.compactions ??= [];
+	live.compactions.push(status);
+}
+
+/** The status of compaction task `taskId`, if listed. */
+export function compactionStatus(live: Draft<LiveState>, taskId: TaskId): Draft<CompactionStatus> | undefined {
+	return live.compactions?.find((status) => status.taskId === taskId);
+}
+
+/** Remove the status of compaction task `taskId`, and the list once empty. */
+export function removeCompactionStatus(live: Draft<LiveState>, taskId: TaskId): void {
+	const statuses = live.compactions;
+	if (statuses === undefined) return;
+	const index = statuses.findIndex((status) => status.taskId === taskId);
+	if (index >= 0) statuses.splice(index, 1);
+	if (statuses.length === 0) delete live.compactions;
+}
+
+/** Remove the status of compaction task `taskId` and record that it failed with `error`. */
+export function failCompactionStatus(live: Draft<LiveState>, taskId: TaskId, error: string): void {
+	const status = compactionStatus(live, taskId);
+	if (status !== undefined) live.compactionFailure = { taskId: status.taskId, reason: status.reason, error };
+	removeCompactionStatus(live, taskId);
+}
+
+/**
+ * The slot of tool task `taskId`, model-issued or nested, if `pi.live` still lists it. Nested slots are found by binary
+ * search: a tool may make thousands of nested calls, each of which looks up its slot in several commits, and every
+ * element a search reads costs a node in the draft's change tracker. Their task IDs ascend: each slot is appended in
+ * the commit that creates its task, task IDs ascend in commit order, and removal keeps the order.
+ */
+export function toolSlot(live: Draft<LiveState>, taskId: TaskId): Draft<ToolSlot | NestedToolSlot> | undefined {
+	const slot = live.tools?.find((each) => each.taskId === taskId);
+	if (slot !== undefined || live.nestedTools === undefined) return slot;
+	const nested = live.nestedTools;
+	let low = 0;
+	let high = nested.length - 1;
+	while (low <= high) {
+		const middle = (low + high) >>> 1;
+		const candidate = nested[middle]!;
+		if (candidate.taskId === taskId) return candidate;
+		if (candidate.taskId < taskId) low = middle + 1;
+		else high = middle - 1;
+	}
+	return undefined;
+}
+
+/**
+ * Mark a slot done and clear its running output; the caller sets a model-issued call's result `entry`, or a nested
+ * call's `summary`. Neither is set when the task faulted or was orphaned.
+ */
+export function finishSlot(slot: Draft<ToolSlot | NestedToolSlot>): void {
+	slot.status = "done";
+	clearProgress(slot);
+}
+
+/** Remove the nested slots below tool task `taskId`, transitively; the list holds parents before their children. */
+export function removeNestedSlots(live: Draft<LiveState>, taskId: TaskId): void {
+	const nested = live.nestedTools;
+	if (nested === undefined) return;
+	const removed = new Set<TaskId>([taskId]);
+	for (const slot of nested) if (removed.has(slot.parentTaskId)) removed.add(slot.taskId);
+	for (let index = nested.length - 1; index >= 0; index--) {
+		if (removed.has(nested[index]!.parentTaskId)) nested.splice(index, 1);
+	}
+	if (nested.length === 0) delete live.nestedTools;
+}
+
+/** Remove what a tool published while running; its result entry or a rerun replaces it. */
+export function clearProgress(slot: Draft<SlotProgress>): void {
+	delete slot.output;
+	delete slot.droppedBytes;
+	delete slot.droppedLines;
+	delete slot.details;
+	delete slot.diagnostics;
+}
+
+/**
+ * Harness cleanup for a terminal outcome the scheduler writes itself (`faulted` or `orphaned`). A run task ends its
+ * run; a tool task's slot is marked done without an entry, and context derivation synthesizes the missing result; a
+ * compaction task's status is removed and its failure recorded.
+ * Ignores other kinds so it never creates `pi.live` elsewhere. The scheduler calls this without knowing task kinds;
+ * the Harness passes it in (spec §5.4).
+ * REMINDER: a committed generation partial becomes an aborted assistant entry here, exactly as in the generation abort
+ * handler, so the transcript keeps what the model produced and `pi.usage` counts its spend. The scheduler's commit has
+ * no task scope, so that entry has no `byTaskId`. Faults come from task bugs
+ * or malformed provider data (a non-JSON value in a response), or a commit the Storage rejected without effect; an
+ * uncertain storage failure poisons the Session instead and writes no outcome.
+ */
+export async function settleSchedulerOutcome(
+	tx: Transaction,
+	record: TaskRecord<JsonValue, JsonValue, JsonValue>,
+	outcome: SchedulerOutcome,
+): Promise<void> {
+	if (record.kind === TOOL_TASK_KIND) {
+		const live = await tx.doc(LiveDoc, record.conversationId);
+		const slot = toolSlot(live, record.id);
+		if (slot !== undefined) finishSlot(slot);
+		removeNestedSlots(live, record.id);
+		return;
+	}
+	if (record.kind === COMPACTION_TASK_KIND) {
+		const error = outcome.status === "faulted" ? outcome.error.message : outcome.reason;
+		failCompactionStatus(await tx.doc(LiveDoc, record.conversationId), record.id, error);
+		return;
+	}
+	if (!RUN_TASK_KINDS.has(record.kind)) return;
+	const live = await tx.doc(LiveDoc, record.conversationId);
+	if (live.run?.taskId !== record.id) return;
+	await convertPartial(tx, live, record.conversationId);
+	endRun(
+		tx,
+		live,
+		record.id,
+		outcome.status === "faulted"
+			? { status: "unanswered", reason: "faulted", detail: outcome.error.message }
+			: { status: "unanswered", reason: outcome.reason },
+	);
+}

third_party/pi/packages/durable/src/harness/prompt.ts

@@ -0,0 +1,158 @@
+import type { Context } from "@earendil-works/chord";
+import type { Message, SystemMessage, Tool, ToolReference } from "@earendil-works/pi-ai";
+import { declarationsEqual, getCurrentTools, toToolDeclaration } from "@earendil-works/pi-ai/utils/transcript";
+import { SystemEntry } from "../entries.ts";
+import type { ContextEdit, TypedEntryDraft } from "../types.ts";
+import type { ContextView, PromptInput, PromptSection, ToolRegistration } from "./types.ts";
+
+/** Sections in effect after replaying system messages in order: set in place, `null` deletes, re-adding appends. */
+export function replaySections(messages: readonly Message[]): Map<string, string> {
+	const shown = new Map<string, string>();
+	for (const message of messages) {
+		if (message.role !== "system" || message.sections === undefined) continue;
+		for (const [key, value] of Object.entries(message.sections)) {
+			if (value === null) shown.delete(key);
+			else shown.set(key, value);
+		}
+	}
+	return shown;
+}
+
+/**
+ * Render the agent's sections in order. `undefined` omits a section; tagged text is wrapped as `<key>\n...\n</key>`. A
+ * section that throws keeps its shown text, if any, and is reported; errors after `context` is aborted propagate.
+ */
+export async function renderSections<Tool extends ToolRegistration>(
+	sections: readonly PromptSection<Tool>[],
+	input: PromptInput<Tool>,
+	shown: ReadonlyMap<string, string>,
+	report: (error: unknown) => void,
+	context: Context,
+): Promise<Map<string, string>> {
+	const desired = new Map<string, string>();
+	for (const section of sections) {
+		let text: string | undefined;
+		try {
+			text = await section.render(input, context);
+		} catch (error) {
+			if (context.abortSignal?.aborted) throw error;
+			report(error);
+			const kept = shown.get(section.key);
+			if (kept !== undefined) desired.set(section.key, kept);
+			continue;
+		}
+		if (text === undefined) continue;
+		desired.set(section.key, section.tag === false ? text : `<${section.key}>\n${text}\n</${section.key}>`);
+	}
+	return desired;
+}
+
+type SystemDraft = TypedEntryDraft<never>;
+
+type ToolChanges = { readonly toolsRemoved: ToolReference[]; readonly toolsAdded: Tool[] };
+
+/**
+ * Plan the `pi.system` entries that make the replayed sections and tools of `view` equal `desired` and `tools` in values
+ * and order.
+ *
+ * - A head marker with no later `pi.system` entry in context: one complete baseline that omits every retained earlier
+ *   `pi.system` entry, written even when it restates the replayed values.
+ * - Otherwise, when a minimal section patch would leave a different order: remove every shown section, then re-add
+ *   every desired section in order.
+ * - Otherwise the minimal patch of changed values and `null` removals, or nothing.
+ *
+ * Tool changes ride on the last planned entry, or on one entry of their own.
+ */
+export function planSystemEntries(
+	view: ContextView,
+	desired: ReadonlyMap<string, string>,
+	tools: readonly Tool[],
+	timestamp: number,
+): SystemDraft[] {
+	const head = view.head;
+	if (head !== undefined && !view.entries.some((entry) => SystemEntry.is(entry) && entry.id > head.id)) {
+		const edits: ContextEdit[] = view.entries
+			.filter((entry) => SystemEntry.is(entry))
+			.map((entry) => ({ target: entry.id, action: "omit" }));
+		const baseline = systemEntry(
+			Object.fromEntries(desired),
+			{ toolsRemoved: [], toolsAdded: tools.map(toToolDeclaration) },
+			timestamp,
+		);
+		return [edits.length === 0 ? baseline : { ...baseline, edits }];
+	}
+	const sections = planSections(replaySections(view.messages), desired);
+	const changes = planTools(getCurrentTools(view.messages), tools);
+	if (changes.toolsRemoved.length === 0 && changes.toolsAdded.length === 0) {
+		return sections.map((patch) => systemEntry(patch, undefined, timestamp));
+	}
+	if (sections.length === 0) return [systemEntry(undefined, changes, timestamp)];
+	return sections.map((patch, index) =>
+		systemEntry(patch, index === sections.length - 1 ? changes : undefined, timestamp),
+	);
+}
+
+/**
+ * Tool changes from `offered` to `desired`. A changed declaration is removed and re-added. Replay keeps retained tools
+ * in place and appends additions; when that would not yield the desired order, every offered tool is removed and
+ * every desired tool re-added in order.
+ */
+function planTools(offered: readonly Tool[], desired: readonly Tool[]): ToolChanges {
+	const wanted = new Map(desired.map((tool) => [tool.name, tool]));
+	const kept = offered.filter((tool) => {
+		const next = wanted.get(tool.name);
+		return next !== undefined && declarationsEqual(tool, next);
+	});
+	const keptNames = new Set(kept.map((tool) => tool.name));
+	const added = desired.filter((tool) => !keptNames.has(tool.name));
+	const replayed = [...kept, ...added];
+	if (replayed.some((tool, index) => tool.name !== desired[index]!.name)) {
+		return {
+			toolsRemoved: offered.map((tool) => ({ name: tool.name })),
+			toolsAdded: desired.map(toToolDeclaration),
+		};
+	}
+	return {
+		toolsRemoved: offered.filter((tool) => !keptNames.has(tool.name)).map((tool) => ({ name: tool.name })),
+		toolsAdded: added.map(toToolDeclaration),
+	};
+}
+
+/** Section patches: none, the minimal patch, or a remove-all/re-add-all pair when the order would differ. */
+function planSections(
+	shown: ReadonlyMap<string, string>,
+	desired: ReadonlyMap<string, string>,
+): Record<string, string | null>[] {
+	const patchedOrder = [
+		...[...shown.keys()].filter((key) => desired.has(key)),
+		...[...desired.keys()].filter((key) => !shown.has(key)),
+	];
+	const desiredOrder = [...desired.keys()];
+	if (patchedOrder.some((key, index) => key !== desiredOrder[index])) {
+		return [Object.fromEntries([...shown.keys()].map((key) => [key, null])), Object.fromEntries(desired)];
+	}
+
+	const patch: Record<string, string | null> = {};
+	for (const [key, value] of shown) {
+		const next = desired.get(key);
+		if (next !== value) patch[key] = next ?? null;
+	}
+	for (const [key, value] of desired) if (!shown.has(key)) patch[key] = value;
+	return Object.keys(patch).length === 0 ? [] : [patch];
+}
+
+function systemEntry(
+	sections: Record<string, string | null> | undefined,
+	tools: ToolChanges | undefined,
+	timestamp: number,
+): SystemDraft {
+	const message: SystemMessage = {
+		role: "system",
+		content: "",
+		...(sections === undefined ? {} : { sections }),
+		...(tools === undefined || tools.toolsRemoved.length === 0 ? {} : { toolsRemoved: tools.toolsRemoved }),
+		...(tools === undefined || tools.toolsAdded.length === 0 ? {} : { toolsAdded: tools.toolsAdded }),
+		timestamp,
+	};
+	return { model: [message] };
+}

third_party/pi/packages/durable/src/harness/provider.ts

@@ -0,0 +1,39 @@
+import type { Context } from "@earendil-works/chord";
+import { uuidv7 } from "@earendil-works/pi-ai/utils/uuid";
+import { defineDoc } from "../documents.ts";
+import type { TaskRuntime } from "../types.ts";
+
+/** Stable provider-facing identity of one conversation. */
+export type ProviderState = {
+	sessionId: string;
+};
+
+/** Built-in provider state; every fork starts with a fresh identity instead of copying its parent. */
+export const ProviderDoc = defineDoc<ProviderState>({
+	kind: "pi.provider",
+	version: 1,
+	scope: "conversation",
+	history: "latest",
+	fork: "initial",
+	initial: () => ({ sessionId: uuidv7() }),
+	checkpointWhen: () => true,
+});
+
+/**
+ * Return the persisted identity without writing in the normal path. A legacy conversation without `pi.provider` gets
+ * one migration commit whose `tx.doc()` runs `initial()` before the provider request starts.
+ */
+export async function ensureProviderSessionId<I, S, R, H extends object>(
+	runtime: TaskRuntime<I, S, R, H>,
+	context: Context,
+): Promise<string> {
+	const existing = await runtime.snapshot(ProviderDoc, runtime.conversationId, context);
+	if (existing !== undefined) return existing.sessionId;
+	let created: string | undefined;
+	await runtime.commit(async (tx) => {
+		created = (await tx.doc(ProviderDoc, runtime.conversationId)).sessionId;
+		return undefined;
+	}, context);
+	if (created === undefined) throw new Error(`Conversation ${runtime.conversationId} has no provider session ID`);
+	return created;
+}

third_party/pi/packages/durable/src/harness/registry.ts

@@ -0,0 +1,114 @@
+import { INSTRUCTIONS_KEY } from "./agent.ts";
+import { CompactionTask } from "./compaction.ts";
+import { GenerationTask } from "./generation.ts";
+import { ToolTask } from "./tool.ts";
+import type { AnyTask, Extension, PromptSection, Registry, RegistrySnapshot, ToolRegistration } from "./types.ts";
+
+const SECTION_KEY = /^[a-z][a-z0-9_-]*$/;
+
+/** Built-in task definitions every registry holds; they are not an extension and cannot be removed or replaced. */
+export const BUILTIN_TASKS: readonly AnyTask[] = [GenerationTask, ToolTask, CompactionTask];
+
+/** Immutable published registry state. */
+class RegistryState<Tool extends ToolRegistration> implements RegistrySnapshot<Tool> {
+	readonly #extensions: readonly Extension<Tool>[];
+	readonly #byName: ReadonlyMap<string, Extension<Tool>>;
+	readonly #tasks: ReadonlyMap<string, AnyTask>;
+
+	constructor(extensions: readonly Extension<Tool>[]) {
+		this.#extensions = extensions;
+		this.#byName = new Map(extensions.map((extension) => [extension.name, extension]));
+		const tasks = new Map<string, AnyTask>();
+		for (const task of BUILTIN_TASKS) tasks.set(task.definition.name, task);
+		for (const extension of extensions) {
+			for (const task of extension.tasks ?? []) {
+				const name = task.definition.name;
+				if (tasks.has(name)) throw new Error(`Task ${name} of extension ${extension.name} is already installed`);
+				tasks.set(name, task);
+			}
+		}
+		this.#tasks = tasks;
+	}
+
+	installed(): readonly Extension<Tool>[] {
+		return this.#extensions;
+	}
+
+	extension(name: string): Extension<Tool> | undefined {
+		return this.#byName.get(name);
+	}
+
+	tools(): readonly { readonly extension: Extension<Tool>; readonly tool: Tool }[] {
+		return this.#extensions.flatMap((extension) => (extension.tools ?? []).map((tool) => ({ extension, tool })));
+	}
+
+	sections(): readonly { readonly extension: Extension<Tool>; readonly section: PromptSection<Tool> }[] {
+		return this.#extensions.flatMap((extension) =>
+			(extension.sections ?? []).map((section) => ({ extension, section })),
+		);
+	}
+
+	tasks(): readonly AnyTask[] {
+		return [...this.#tasks.values()];
+	}
+
+	task(name: string): AnyTask | undefined {
+		return this.#tasks.get(name);
+	}
+}
+
+class RegistryImpl<Tool extends ToolRegistration> implements Registry<Tool> {
+	#current = new RegistryState<Tool>([]);
+	readonly #listeners = new Set<() => void>();
+
+	snapshot(): RegistrySnapshot<Tool> {
+		return this.#current;
+	}
+
+	subscribe(listener: () => void): () => void {
+		this.#listeners.add(listener);
+		return () => this.#listeners.delete(listener);
+	}
+
+	install(extension: Extension<Tool>): void {
+		validateExtension(extension);
+		const current = this.#current.installed();
+		const index = current.findIndex((installed) => installed.name === extension.name);
+		const next =
+			index < 0 ? [...current, extension] : current.map((installed, at) => (at === index ? extension : installed));
+		this.#publish(next);
+	}
+
+	uninstall(extension: Extension): void {
+		const current = this.#current.installed();
+		if (!current.some((installed) => installed.name === extension.name)) return;
+		this.#publish(current.filter((installed) => installed.name !== extension.name));
+	}
+
+	/** Build and validate the next state, which throws on a task name collision, then publish it synchronously. */
+	#publish(extensions: readonly Extension<Tool>[]): void {
+		this.#current = new RegistryState(extensions);
+		for (const listener of [...this.#listeners]) listener();
+	}
+}
+
+/** Unique tool names and section keys within one extension; valid, unreserved section keys. */
+function validateExtension(extension: Extension): void {
+	const tools = new Set<string>();
+	for (const tool of extension.tools ?? []) {
+		if (tools.has(tool.name)) throw new Error(`Extension ${extension.name} has two tools named ${tool.name}`);
+		tools.add(tool.name);
+	}
+	const sections = new Set<string>();
+	for (const { key } of extension.sections ?? []) {
+		if (!SECTION_KEY.test(key)) throw new TypeError(`Section key ${JSON.stringify(key)} must match ${SECTION_KEY}`);
+		if (key === INSTRUCTIONS_KEY) throw new Error(`Section key ${key} is reserved for the agent's instructions`);
+		if (sections.has(key)) throw new Error(`Extension ${extension.name} has two sections with key ${key}`);
+		sections.add(key);
+	}
+}
+
+/** Create an application-owned registry holding only the built-in tasks. */
+export function createRegistry<Tool extends ToolRegistration = ToolRegistration>(): Registry<Tool> {
+	return new RegistryImpl<Tool>();
+}

third_party/pi/packages/durable/src/harness/submissions.ts

@@ -0,0 +1,241 @@
+import { type Context, copyJson, type JsonRepresentation } from "@earendil-works/chord";
+import { ConversationBusy } from "../errors.ts";
+import type { SessionImpl } from "../session/session.ts";
+import type {
+	CommitPublication,
+	ConversationId,
+	JsonObject,
+	Storage,
+	SubmissionId,
+	SubmissionRecord,
+	Tx,
+} from "../types.ts";
+import { startRun } from "./generation.ts";
+import {
+	appendInput,
+	applyBoundary,
+	hasQueuedReset,
+	InboxDoc,
+	isDisplayOnly,
+	isStale,
+	pickInputEntry,
+	prepareBoundary,
+	type QueueModes,
+	removeInboxItem,
+} from "./inbox.ts";
+import { LiveDoc } from "./live.ts";
+import type { InputEntry, SettledSubmissionRecord, Submission, SubmissionDraft, UserInput } from "./types.ts";
+import { closedError, Waiters } from "./util.ts";
+
+type AbortResult = "aborted" | "already_placed" | "settled";
+
+/** Admission, waits, and withdrawal of the durable submissions of one Harness. */
+export class Submissions {
+	readonly #session: SessionImpl;
+	readonly #storage: Storage;
+	readonly #now: () => number;
+	/** Read at each admission, on the Session line. */
+	readonly #queueModes: () => QueueModes;
+	/** Enable task scheduling; submitting or waiting asks for progress. */
+	readonly #resume: () => void;
+	readonly #waiters = new Waiters<SubmissionId, SettledSubmissionRecord>();
+	#closed = false;
+
+	constructor(
+		session: SessionImpl,
+		storage: Storage,
+		now: () => number,
+		queueModes: () => QueueModes,
+		resume: () => void,
+	) {
+		this.#session = session;
+		this.#storage = storage;
+		this.#now = now;
+		this.#queueModes = queueModes;
+		this.#resume = resume;
+		session.observeCommits((publication) => this.#observe(publication));
+		session.subscribeClose(() => {
+			this.#closed = true;
+			this.#waiters.rejectAll(closedError(this.#session));
+		});
+	}
+
+	/** Admit a submission in one commit; see `admitSubmission()`. */
+	async submit(conversationId: ConversationId, draft: SubmissionDraft, context: Context): Promise<Submission> {
+		this.#resume();
+		const id = await this.#session.commitWith(
+			(tx) => admitSubmission(tx, conversationId, draft, this.#now(), this.#queueModes()),
+			context,
+		);
+		return new SubmissionHandle(id, this);
+	}
+
+	/** Handle for an existing submission, or `undefined`. */
+	async get(id: SubmissionId, context: Context): Promise<Submission | undefined> {
+		const record = await this.#session.readOnLine(() => this.#storage.submission(id, context));
+		return record === undefined ? undefined : new SubmissionHandle(record.id, this);
+	}
+
+	async status(id: SubmissionId, context: Context): Promise<SubmissionRecord> {
+		const record = await this.#session.readOnLine(() => this.#storage.submission(id, context));
+		if (record === undefined) throw new Error(`Submission ${id} does not exist`);
+		return record;
+	}
+
+	async wait(id: SubmissionId, context: Context): Promise<SettledSubmissionRecord> {
+		this.#resume();
+		// Check and register on the line so no settling publication falls between them.
+		const found = await this.#session.readOnLine(async () => {
+			const record = await this.#storage.submission(id, context);
+			if (record === undefined) throw new Error(`Submission ${id} does not exist`);
+			if (isSettled(record)) return { promise: Promise.resolve(record) };
+			// Close rejects registered waiters synchronously and may begin during the read.
+			if (this.#closed) throw closedError(this.#session);
+			return { promise: this.#waiters.add(id, context) };
+		});
+		return found.promise;
+	}
+
+	/** Withdraw a queued submission and remove its inbox item; placed inputs and settled submissions are reported. */
+	abort(id: SubmissionId, context: Context, conversationId?: ConversationId): Promise<AbortResult | "not_found"> {
+		return this.#session.commitWith(async (tx) => {
+			const record = await tx.submission(id);
+			if (record === undefined || (conversationId !== undefined && record.conversationId !== conversationId)) {
+				return "not_found";
+			}
+			if (record.status === "queued") {
+				tx.settleSubmission(id, { status: "unanswered", reason: "aborted" });
+				await removeInboxItem(tx, record.conversationId, id);
+				return "aborted";
+			}
+			return record.status === "placed" ? "already_placed" : "settled";
+		}, context);
+	}
+
+	#observe(publication: CommitPublication): void {
+		for (const change of publication.changes) {
+			if (change.type !== "submission" || !isSettled(change.value)) continue;
+			this.#waiters.resolve(change.value.id, change.value);
+		}
+	}
+}
+
+class SubmissionHandle implements Submission {
+	readonly id: SubmissionId;
+	readonly #submissions: Submissions;
+
+	constructor(id: SubmissionId, submissions: Submissions) {
+		this.id = id;
+		this.#submissions = submissions;
+	}
+
+	status(context: Context): Promise<SubmissionRecord> {
+		return this.#submissions.status(this.id, context);
+	}
+
+	wait(context: Context): Promise<SettledSubmissionRecord> {
+		return this.#submissions.wait(this.id, context);
+	}
+
+	async abort(context: Context): Promise<AbortResult> {
+		const result = await this.#submissions.abort(this.id, context);
+		if (result === "not_found") throw new Error(`Submission ${this.id} does not exist`);
+		return result;
+	}
+}
+
+function isSettled(record: SubmissionRecord): record is SettledSubmissionRecord {
+	return record.status === "done" || record.status === "unanswered";
+}
+
+/** Reject an input entry an untyped caller got wrong: its kind must be a non-empty string outside the `pi.` prefix. */
+function checkInputEntry(entry: InputEntry): void {
+	const { kind } = entry;
+	if (typeof kind !== "string" || kind.length === 0 || kind.startsWith("pi.")) {
+		throw new TypeError(
+			`An input entry's kind must be a non-empty string not starting with "pi.", got ${String(kind)}`,
+		);
+	}
+}
+
+/**
+ * Admit a submission inside a commit (spec §6); `Conversation.submit()` and conversation-owned compactions share it. A
+ * known request ID returns its existing submission without writing. A display-only write (only `kind` and `data`) is
+ * appended and settled `done` at once, busy or not, as it changes nothing the model sees; only behind a queued reset it
+ * queues, so it lands after the reset instead of before it in the context the reset ends. A busy conversation queues anything else in
+ * `pi.inbox`, or rejects `whenBusy: "reject"` input with `ConversationBusy`. An idle conversation with queued items
+ * queues it behind them and runs a final boundary. Otherwise idle input places its entry (`pi.user`, or the input's
+ * own) and starts a run, and an idle write appends its entry and settles `done`, or `stale` when its head reaches
+ * before the active range.
+ */
+export async function admitSubmission(
+	tx: Tx,
+	conversationId: ConversationId,
+	draft: SubmissionDraft,
+	now: number,
+	queueModes: QueueModes,
+): Promise<SubmissionId> {
+	if (draft.type === "input" && draft.entry !== undefined) checkInputEntry(draft.entry);
+	if (draft.requestId !== undefined) {
+		const existing = await tx.submissionByRequest(conversationId, draft.requestId);
+		if (existing !== undefined) {
+			if (existing.type !== draft.type) {
+				throw new Error(`Request ${draft.requestId} already identifies a submission of type ${existing.type}`);
+			}
+			return existing.id;
+		}
+	}
+	const requestId = draft.requestId === undefined ? {} : { requestId: draft.requestId };
+	if (
+		draft.type === "write" &&
+		isDisplayOnly(draft.entry) &&
+		!hasQueuedReset((await tx.doc(InboxDoc, conversationId)).items)
+	) {
+		const entry = await tx.appendEntry(conversationId, draft.entry);
+		const write = { conversationId, ...requestId, type: "write", status: "done", entry: entry.id } as const;
+		return (await tx.createSubmission(write)).id;
+	}
+	const live = await tx.doc(LiveDoc, conversationId);
+	const busy = live.run !== undefined;
+	if (busy && draft.type === "input" && draft.whenBusy === "reject") throw new ConversationBusy(conversationId);
+	// A boundary reads the table, so it is prepared before the first table write; a busy one needs none.
+	const boundary = busy ? undefined : await prepareBoundary(tx, conversationId, queueModes);
+	if (boundary === undefined || boundary.inbox.items.length > 0) {
+		const { id } = await tx.createSubmission({
+			conversationId,
+			...requestId,
+			type: draft.type,
+			status: "queued",
+		});
+		// Hosts may leave optional fields `undefined`; drafts take strict JSON.
+		const value = copyJson(draft.type === "write" ? draft.entry : draft.content, {
+			omitUndefinedProperties: true,
+		});
+		const items = (boundary?.inbox ?? (await tx.doc(InboxDoc, conversationId))).items;
+		if (draft.type === "write") items.push({ id, mode: "write", entry: value as JsonObject });
+		else {
+			const mode = draft.whenBusy === "steer" ? "steer" : "followUp";
+			const content = value as JsonRepresentation<UserInput>;
+			const entry = draft.entry === undefined ? {} : { entry: pickInputEntry(draft.entry) };
+			items.push({ id, mode, content, ...entry });
+		}
+		if (boundary === undefined) return id;
+		const { users } = await applyBoundary(tx, boundary, "final", now);
+		if (users.length > 0) await startRun(tx, conversationId, live, users);
+		return id;
+	}
+	if (draft.type === "write") {
+		if (isStale(boundary, draft.entry)) {
+			const stale = { status: "unanswered", reason: "stale" } as const;
+			return (await tx.createSubmission({ conversationId, ...requestId, type: "write", ...stale })).id;
+		}
+		const entry = await tx.appendEntry(conversationId, draft.entry);
+		const write = { conversationId, ...requestId, type: "write", status: "done", entry: entry.id } as const;
+		return (await tx.createSubmission(write)).id;
+	}
+	const entry = await appendInput(tx, conversationId, draft.content, draft.entry, now);
+	const input = { conversationId, ...requestId, type: "input", status: "placed", entry: entry.id } as const;
+	const { id } = await tx.createSubmission(input);
+	await startRun(tx, conversationId, live, [id]);
+	return id;
+}

third_party/pi/packages/durable/src/harness/task-graph.ts

@@ -0,0 +1,236 @@
+import { type AttachedReplicatedState, type Context, type JsonValue, replicatedState } from "@earendil-works/chord";
+import { withoutAbortSignal } from "@earendil-works/chord/context";
+import { applyImmutable, type Op } from "@earendil-works/chord/delta";
+import { CommittedStateSource, CommittedWatch } from "../session/observation.ts";
+import { contained, type SessionImpl } from "../session/session.ts";
+import type {
+	CommitPublication,
+	ConversationId,
+	JoinPolicy,
+	Storage,
+	TaskId,
+	TaskOutcome,
+	TaskRecord,
+	WatchHandle,
+} from "../types.ts";
+import { closedError, scanAll } from "./util.ts";
+
+/** A live task's durable status without its checkpoint and outcome payloads (spec §9.5). */
+export type TaskGraphState =
+	| { readonly status: "pending" | "running"; readonly phase: string }
+	| {
+			readonly status: "waiting";
+			readonly phase: string;
+			readonly on: readonly TaskId[];
+			readonly policy: JoinPolicy;
+	  }
+	/** Outcome held until its ordinary owned work drains. */
+	| { readonly status: "completing"; readonly outcome: TaskOutcome<JsonValue>["status"] };
+
+export type TaskGraphNode = {
+	readonly id: TaskId;
+	readonly kind: string;
+	readonly conversationId: ConversationId;
+	/** Owner task; absent for a conversation-owned task. */
+	readonly owner?: TaskId;
+	readonly background: boolean;
+	readonly abortRequested: boolean;
+	readonly state: TaskGraphState;
+	/** Conversations this task owns, in ID order. */
+	readonly conversations: readonly ConversationId[];
+};
+
+/** Every live task of the Session (spec §9.5). */
+export type TaskGraph = {
+	/** Every live task, keyed by its decimal ID. */
+	readonly tasks: Readonly<Record<string, TaskGraphNode>>;
+};
+
+export type TaskGraphWatch = WatchHandle<TaskGraph>;
+
+type AnyTaskRecord = TaskRecord<JsonValue, JsonValue, JsonValue>;
+type Observer = CommittedStateSource<TaskGraph> | CommittedWatch<TaskGraph>;
+type Mount = { value: TaskGraph; readonly observers: Set<Observer> };
+
+const LIVE_STATUSES = ["pending", "running", "waiting", "completing"] as const;
+const SCAN_PAGE_SIZE = 256;
+
+/**
+ * The Harness's task graph mount: built on the Session line by its first observer and dropped with its last. It
+ * advances from the Session's commit publications, which are durable.
+ */
+export class TaskGraphView {
+	readonly #session: SessionImpl;
+	readonly #storage: Storage;
+	#mount: Mount | undefined;
+	#closed = false;
+
+	constructor(session: SessionImpl, storage: Storage) {
+		this.#session = session;
+		this.#storage = storage;
+		session.observeCommits((publication, context) => {
+			if (this.#mount !== undefined) advance(this.#mount, publication, context, (error) => session.report(error));
+		});
+		session.subscribeClose(() => {
+			this.#closed = true;
+			for (const observer of [...(this.#mount?.observers ?? [])]) {
+				contained(
+					() => observer.closeSession(this.#session.failure),
+					(error) => session.report(error),
+				);
+			}
+			this.#mount = undefined;
+		});
+	}
+
+	/** A disposable read-only Chord state of the graph. */
+	async state(context: Context): Promise<AttachedReplicatedState<TaskGraph>> {
+		const { observer, detach } = await this.#attach(
+			(value, release) => new CommittedStateSource<TaskGraph>(value, release),
+			context,
+		);
+		try {
+			return replicatedState(observer, { onError: (error) => this.#session.report(error) });
+		} catch (error) {
+			detach();
+			throw error;
+		}
+	}
+
+	/** A serialized exact-frame watch of the graph; cancelling `context` stops it. */
+	async watch(context: Context): Promise<TaskGraphWatch> {
+		const { observer } = await this.#attach(
+			(value, release) => new CommittedWatch<TaskGraph>(value, release, (error) => this.#session.report(error)),
+			context,
+		);
+		const signal = context.abortSignal;
+		if (signal?.aborted) {
+			observer.cancel();
+			throw signal.reason;
+		}
+		if (signal !== undefined) observer.observeCancellation(signal);
+		return observer;
+	}
+
+	/** Register an observer created from the current revision, atomically on the Session line. */
+	#attach<O extends Observer>(
+		create: (value: TaskGraph, release: () => void) => O,
+		context: Context,
+	): Promise<{ observer: O; detach: () => void }> {
+		return this.#session.readOnLine(async () => {
+			const mount = this.#mount ?? { value: await this.#build(context), observers: new Set<Observer>() };
+			const detach = (): void => {
+				mount.observers.delete(observer);
+				if (mount.observers.size === 0 && this.#mount === mount) this.#mount = undefined;
+			};
+			const observer = create(mount.value, detach);
+			// Close or cancellation may begin while the mount builds; register nothing then.
+			if (this.#closed) throw closedError(this.#session);
+			context.abortSignal?.throwIfAborted();
+			this.#mount = mount;
+			mount.observers.add(observer);
+			return { observer, detach };
+		});
+	}
+
+	async #build(context: Context): Promise<TaskGraph> {
+		const tasks: Record<string, TaskGraphNode> = {};
+		const records: AnyTaskRecord[] = [];
+		for (const status of LIVE_STATUSES) {
+			records.push(
+				...(await scanAll((cursor) => this.#storage.scanTasks({ status }, SCAN_PAGE_SIZE, cursor, context))),
+			);
+		}
+		records.sort((a, b) => a.id - b.id);
+		for (const record of records) {
+			const owned = await scanAll((cursor) =>
+				this.#storage.scanConversations({ ownerTaskId: record.id }, SCAN_PAGE_SIZE, cursor, context),
+			);
+			const conversations = owned.map((conversation) => conversation.id).sort((a, b) => a - b);
+			tasks[String(record.id)] = nodeOf(record, conversations);
+		}
+		return { tasks };
+	}
+}
+
+/**
+ * Derive the mount's operations from one publication, apply them, and hand the revision to every observer; one that
+ * throws is reported through `report`, and the others still get it.
+ */
+function advance(
+	mount: Mount,
+	publication: CommitPublication,
+	context: Context,
+	report: (error: unknown) => void,
+): void {
+	const ops: Op[] = [];
+	// Nodes this publication set or deleted, over the mount's value.
+	const changed = new Map<string, TaskGraphNode | undefined>();
+	const node = (key: string): TaskGraphNode | undefined =>
+		changed.has(key) ? changed.get(key) : mount.value.tasks[key];
+	for (const change of publication.changes) {
+		if (change.type !== "task") continue;
+		const record = change.value;
+		const key = String(record.id);
+		const previous = node(key);
+		if (record.state.status === "terminal") {
+			if (previous === undefined) continue;
+			ops.push(["d", ["tasks", key]]);
+			changed.set(key, undefined);
+			continue;
+		}
+		const next = nodeOf(record, previous?.conversations ?? []);
+		if (previous !== undefined && JSON.stringify(next) === JSON.stringify(previous)) continue;
+		ops.push(["s", ["tasks", key], next as unknown as JsonValue]);
+		changed.set(key, next);
+	}
+	// After the tasks, so a conversation created with its owner task in one commit finds the owner's node. Change order
+	// within a publication is unspecified, so each owner's list is sorted again.
+	const created = new Map<string, ConversationId[]>();
+	for (const change of publication.changes) {
+		if (change.type !== "conversation" || change.value.owner === undefined) continue;
+		const key = String(change.value.owner.taskId);
+		if (node(key) !== undefined) created.set(key, [...(created.get(key) ?? []), change.value.id]);
+	}
+	for (const [key, ids] of created) {
+		const conversations = [...node(key)!.conversations, ...ids].sort((a, b) => a - b);
+		ops.push(["s", ["tasks", key, "conversations"], conversations]);
+	}
+	if (ops.length === 0) return;
+	mount.value = applyImmutable(mount.value, ops);
+	const frameContext = withoutAbortSignal(context);
+	for (const observer of [...mount.observers])
+		contained(() => observer.advance(mount.value, ops, frameContext), report);
+}
+
+function nodeOf(record: AnyTaskRecord, conversations: readonly ConversationId[]): TaskGraphNode {
+	return {
+		id: record.id,
+		kind: record.kind,
+		conversationId: record.conversationId,
+		...(record.owner === undefined ? {} : { owner: record.owner }),
+		background: record.background,
+		abortRequested: record.abortRequested,
+		state: stateOf(record),
+		conversations,
+	};
+}
+
+function stateOf(record: AnyTaskRecord): TaskGraphState {
+	const state = record.state;
+	switch (state.status) {
+		case "pending":
+		case "running":
+			return { status: state.status, phase: phaseOf(state.checkpoint) };
+		case "waiting":
+			return { status: "waiting", phase: phaseOf(state.checkpoint), on: [...state.on], policy: state.policy };
+		// Terminal records never reach here: they leave the graph.
+		case "completing":
+		case "terminal":
+			return { status: "completing", outcome: state.outcome.status };
+	}
+}
+
+function phaseOf(checkpoint: JsonValue): string {
+	return (checkpoint as { readonly phase: string }).phase;
+}

third_party/pi/packages/durable/src/harness/tool.ts

@@ -0,0 +1,911 @@
+import { type Context, copyJson, type Draft, type JsonRepresentation, type JsonValue } from "@earendil-works/chord";
+import { awaitWithContext } from "@earendil-works/chord/context";
+import { overlap } from "@earendil-works/chord/delta";
+import {
+	boundOutput,
+	OutputBuffer,
+	type OutputLimits,
+	PROGRESS_BYTES_PER_SECOND,
+	Progress,
+} from "@earendil-works/chord/output";
+import type { ImageContent, TextContent, ToolCall, ToolResultMessage } from "@earendil-works/pi-ai";
+import { validateToolArguments } from "@earendil-works/pi-ai/utils/validation";
+import { Compile } from "typebox/compile";
+import { defineDocFamily } from "../documents.ts";
+import { AssistantEntry, ToolResultEntry } from "../entries.ts";
+import { errorMessage } from "../errors.ts";
+import { defineTask } from "../tasks.ts";
+import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, utf8ByteLength } from "../truncate.ts";
+import type {
+	ConversationId,
+	EntryId,
+	JsonObject,
+	Task,
+	TaskId,
+	TaskOptions,
+	TaskOutcome,
+	TaskRuntime,
+	Tx,
+	TypedEntry,
+} from "../types.ts";
+import { assignJson } from "./json.ts";
+import {
+	clearProgress,
+	finishSlot,
+	LiveDoc,
+	type NestedToolSlot,
+	type NestedToolSummary,
+	removeNestedSlots,
+	type ToolSlot,
+	toolSlot,
+} from "./live.ts";
+import type {
+	NestedToolExecutionResult,
+	ToolControl,
+	ToolDiagnostic,
+	ToolExecutionApi,
+	ToolExecutionResult,
+	ToolHookCall,
+	ToolHooks,
+	ToolRegistration,
+} from "./types.ts";
+import { recordUsage } from "./usage.ts";
+
+/**
+ * A model-issued call, read from its assistant entry, or a nested call a running tool made through `executeTool()`,
+ * whose call the input carries because no entry holds it.
+ */
+export type ToolTaskInput =
+	| { kind: "model"; assistant: EntryId; callId: string }
+	| {
+			kind: "nested";
+			parent: TaskId;
+			parentCallId: string;
+			key: string;
+			call: ToolCall;
+			/** `false`: commit no running output, details, or diagnostics to the slot; the result still gets them. */
+			progress?: false;
+	  };
+
+export type ToolTaskCheckpoint =
+	| { phase: "call" }
+	/** Durable intent: the final arguments and the replay policy recorded before execution. */
+	| { phase: "execute"; arguments: JsonObject; replay: "safe" | "unsafe" };
+
+/**
+ * A model-issued call's result is its transcript entry; a nested call's result is in its caller's
+ * `NestedCallDoc`, so the receipt stays small and the result retires with the caller. A version 1 task that was
+ * already holding its outcome as `completing` when the Harness upgraded finishes with a version 1 result,
+ * `{ entryId, control? }` without `kind`; treat a result without `kind` as a model-issued call's.
+ */
+export type ToolTaskResult = { kind: "model"; entryId: EntryId; control?: ToolControl } | { kind: "nested" };
+
+/**
+ * Memo of a tool task that made a nested call, in any attempt, committed before the first admission of each attempt:
+ * only then can `pi.live` hold slots below it, which settling removes. A crash between the two only costs a scan.
+ */
+const MADE_NESTED_CALLS = "pi.tool.madeNestedCalls";
+
+/** One nested call of a running tool: its task, and once it ended, its result. */
+export type NestedCallState = {
+	/** Absent only inside the commit that creates the call; a creation that fails leaves no member. */
+	taskId?: TaskId<ToolTaskResult>;
+	/** Exactly what `executeTool()` returns; absent while running, and for a call the scheduler faulted or orphaned. */
+	result?: JsonRepresentation<NestedToolExecutionResult>;
+};
+
+/**
+ * The nested calls of a tool call, a task-scoped family of the calling task keyed by the nested call's key: so a rerun
+ * finds the calls it already made, and the caller reads each result. Created with the nested task; the nested call's
+ * terminal commit adds its result. One document per call, so no commit rewrites the others. Retires with the caller.
+ */
+export const NestedCallDoc = defineDocFamily<NestedCallState, null>({
+	kind: "pi.tool.nested-call",
+	version: 1,
+	scope: "task",
+	family: true,
+	initial: () => ({}),
+});
+
+type Runtime = TaskRuntime<ToolTaskInput, ToolTaskCheckpoint, ToolTaskResult, ToolHooks>;
+type Content = (TextContent | ImageContent)[];
+
+/**
+ * Built-in tool task: resolves the called tool among its phase agent's tools, validates, runs `beforeTool`, records intent,
+ * executes, runs `afterTool`, and settles the result, all in one `call` handler so nothing separates resolution from
+ * settlement. `execute` is reached only by recovery and applies the replay rule. A model-issued call settles by
+ * appending its result entry; a nested call by writing its result to its caller's `NestedCallDoc`.
+ */
+export const ToolTask = defineTask<ToolTaskInput, ToolTaskCheckpoint, ToolTaskResult, ToolHooks>({
+	name: "pi.tool",
+	version: 2,
+	initial: () => ({ phase: "call" }),
+	// Version 1 had only model-issued calls.
+	migrate: (input, checkpoint) => ({
+		input: { kind: "model", ...(input as { assistant: EntryId; callId: string }) },
+		checkpoint: checkpoint as ToolTaskCheckpoint,
+	}),
+	phases: {
+		call: async (task, runtime, context) => {
+			const call = await readCall(runtime, task.input, context);
+			const tool = await resolveTool(runtime, task.input, call.name, context);
+			if (tool === undefined) {
+				const error = harnessError("tool_unavailable", `Tool ${call.name} is not available`);
+				return settle(runtime, task.input, call, COMPLETED, () => error, context);
+			}
+			const prepared = prepare(tool, call.arguments as JsonObject);
+			const checked = "error" in prepared ? prepared : validate(tool, call, prepared.args);
+			if ("error" in checked)
+				return settle(runtime, task.input, call, COMPLETED, () => invalid(checked.error), context);
+			let args = checked.args;
+			let block: string | undefined;
+			await runtime.hooks.each("beforeTool", async (hook) => {
+				if (block !== undefined) return;
+				try {
+					const decision = await hook({ ...call, arguments: args }, runtime, context);
+					if (decision?.block !== undefined) block = decision.block;
+					else if (decision?.arguments !== undefined) args = decision.arguments;
+				} catch (error) {
+					if (runtime.signal.aborted) throw error;
+					block = errorMessage(error);
+				}
+			});
+			if (block !== undefined) {
+				const blocked = harnessError("blocked", `Tool call blocked: ${block}`);
+				return settle(runtime, task.input, call, COMPLETED, () => blocked, context);
+			}
+			const validated = validate(tool, call, args);
+			if ("error" in validated)
+				return settle(runtime, task.input, call, COMPLETED, () => invalid(validated.error), context);
+			// Strict JSON: repair may set optional properties to undefined, which the slot and checkpoint cannot hold.
+			const final = copyJson(validated.args, { omitUndefinedProperties: true }) as JsonObject;
+			await runtime.commit(async (tx) => {
+				const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId);
+				if (slot !== undefined) slot.status = "running";
+				// A nested slot shows the arguments the call runs with, which repair, hooks, and coercion may have changed.
+				// Compared with the input, which the slot got at admission: reading the draft would track every leaf.
+				const changed = task.input.kind === "nested" && !jsonEqual(task.input.call.arguments as JsonValue, final);
+				if (changed && slot !== undefined && "parentTaskId" in slot) slot.arguments = final;
+				const intent = { phase: "execute", arguments: final, replay: tool.replay ?? "unsafe" } as const;
+				return { status: "running", checkpoint: intent };
+			}, context);
+			await run(runtime, task.input, call, tool, final, context);
+		},
+		/** Recovery after intent: rerun only when the stored and the current policy both say `safe`. */
+		execute: async (task, runtime, context) => {
+			const { arguments: args, replay } = task.state.checkpoint;
+			const call = await readCall(runtime, task.input, context);
+			const tool = await resolveTool(runtime, task.input, call.name, context);
+			if (replay === "safe" && tool?.replay === "safe") {
+				// The rerun reports from scratch; clear what the interrupted attempt published.
+				await runtime.commit(async (tx) => {
+					const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId);
+					if (slot !== undefined) clearProgress(slot);
+					return undefined;
+				}, context);
+				return run(runtime, task.input, call, tool, args, context);
+			}
+			const message = `Tool ${call.name} was interrupted and may have partially run`;
+			// `failed` records cancellation intent, so the call's owned conversations, left unsupervised, are aborted.
+			const ending = { status: "failed", message } as const;
+			await settle(runtime, task.input, call, ending, (slot) => fromSlot(slot, "interrupted", message), context);
+		},
+	},
+	abort: async (task, runtime, context) => {
+		const call = await readCall(runtime, task.input, context);
+		// Abandoned after a restart (`TaskOptions.abandonOnRestart`): say whether the call may have run.
+		const [code, message] =
+			task.abortReason !== "restart"
+				? ["aborted", `Tool ${call.name} was aborted`]
+				: task.state.checkpoint.phase === "execute"
+					? ["interrupted", `Tool ${call.name} was interrupted by a restart and may have partially run`]
+					: ["abandoned", `Tool ${call.name} was not started: its caller ended with a restart`];
+		const ending = { status: "aborted" } as const;
+		await settle(runtime, task.input, call, ending, (slot) => fromSlot(slot, code, message), context);
+	},
+});
+
+/** The called tool: among the tools the model is offered for a model-issued call, else among those tools may call. */
+async function resolveTool(
+	runtime: Runtime,
+	input: ToolTaskInput,
+	name: string,
+	context: Context,
+): Promise<ToolRegistration | undefined> {
+	const agent = await runtime.agent(context);
+	return (input.kind === "model" ? agent.tools : agent.callable).find((tool) => tool.name === name);
+}
+
+/** The call: from the input for a nested call, which hooks see with its parent, else from the assistant entry. */
+async function readCall(runtime: Runtime, input: ToolTaskInput, context: Context): Promise<ToolHookCall> {
+	if (input.kind === "nested") return { ...input.call, parent: { taskId: input.parent, callId: input.parentCallId } };
+	const entry = await runtime.entry(AssistantEntry, input.assistant, context);
+	const message = entry?.model?.[0];
+	const call =
+		message?.role === "assistant"
+			? message.content.find(
+					(content): content is ToolCall => content.type === "toolCall" && content.id === input.callId,
+				)
+			: undefined;
+	if (call === undefined) throw new Error(`Entry ${input.assistant} has no tool call ${input.callId}`);
+	return call;
+}
+
+/** Arguments, or why they are invalid. */
+type Checked = { readonly args: JsonObject } | { readonly error: string };
+
+/** The call's arguments as repaired by the tool; a throwing repair makes them invalid. */
+function prepare(tool: ToolRegistration, args: JsonObject): Checked {
+	if (tool.prepareArguments === undefined) return { args };
+	try {
+		return { args: tool.prepareArguments(args) as JsonObject };
+	} catch (error) {
+		return { error: errorMessage(error) };
+	}
+}
+
+/** Arguments validated and coerced against the implementation's schema. */
+function validate(tool: ToolRegistration, call: ToolCall, args: JsonObject): Checked {
+	try {
+		return { args: validateToolArguments(tool, { ...call, arguments: args }) as JsonObject };
+	} catch (error) {
+		return { error: errorMessage(error) };
+	}
+}
+
+function invalid(message: string): ToolExecutionResult {
+	return harnessError("invalid_arguments", message);
+}
+
+/** What a running tool reported through its api: output, the last details, and diagnostics. */
+type Reported = {
+	readonly output: OutputBuffer;
+	readonly limits: OutputLimits;
+	readonly diagnostics: ToolDiagnostic[];
+	details: JsonValue | undefined;
+};
+
+/** Execute with the resolved implementation, then settle its result. */
+async function run(
+	runtime: Runtime,
+	input: ToolTaskInput,
+	call: ToolHookCall,
+	tool: ToolRegistration,
+	args: JsonObject,
+	context: Context,
+): Promise<void> {
+	const limits: OutputLimits = {
+		maxBytes: tool.outputLimits?.maxBytes ?? DEFAULT_MAX_BYTES,
+		maxLines: tool.outputLimits?.maxLines ?? DEFAULT_MAX_LINES,
+		retain: tool.outputLimits?.retain ?? "head",
+	};
+	const reported: Reported = { output: new OutputBuffer(limits), limits, diagnostics: [], details: undefined };
+	const streams = input.kind === "model" || input.progress !== false;
+	const progress = publishProgress(runtime, reported, streams, context);
+	/** Nested call admissions this invocation started; cleanup waits for them, so it sees every nested call. */
+	const admissions: Promise<unknown>[] = [];
+	// A caller that can rerun reattaches to its nested calls and child tasks after a restart; any other caller never
+	// resumes, so they are abandoned (`TaskOptions.abandonOnRestart`).
+	const resumes = tool.replay === "safe";
+	/** Default keys: the order of this invocation's calls, so a rerun that calls in the same order reattaches. */
+	let sequence = 0;
+	/** This attempt's commit of the `MADE_NESTED_CALLS` memo. */
+	let madeNestedMemo: Promise<unknown> | undefined;
+	let ended = false;
+	const assertLive = (): void => {
+		if (ended) throw new Error(`Tool call ${call.id} has settled`);
+	};
+	const api: Omit<ToolExecutionApi, "env"> = {
+		taskId: runtime.taskId,
+		conversationId: runtime.conversationId,
+		callId: call.id,
+		registry: runtime.registry,
+		agent: runtime.agent,
+		models: runtime.models,
+		output: (chunk, skipped) => {
+			assertLive();
+			if (reported.output.push(chunk, skipped)) progress.mark();
+		},
+		outputWindow:
+			limits.retain === "tail"
+				? {
+						maxBytes: limits.maxBytes,
+						maxLines: limits.maxLines,
+						minIntervalMs: runtime.settings.progress.outputIntervalMs,
+						bytesPerSecond: PROGRESS_BYTES_PER_SECOND,
+					}
+				: undefined,
+		retainedOutput: () => {
+			assertLive();
+			const retained = reported.output.snapshot();
+			return { text: retained.text, truncated: retained.droppedBytes > 0 };
+		},
+		diagnostic: (diagnostic) => {
+			assertLive();
+			reported.diagnostics.push(copyJson(diagnostic, { omitUndefinedProperties: true }) as ToolDiagnostic);
+			progress.mark();
+		},
+		details: async (value, detailsContext) => {
+			assertLive();
+			detailsContext.abortSignal?.throwIfAborted();
+			reported.details = copyJson(value, { omitUndefinedProperties: true });
+			const committed = progress.markAndWait();
+			// Cancelling the wait leaves the update in place; the commit's own outcome stays observed.
+			committed.catch(() => {});
+			return awaitWithContext(committed, detailsContext);
+		},
+		commit: async (change, commitContext) => {
+			let result: Awaited<ReturnType<typeof change>> | undefined;
+			await runtime.commit(async (tx) => {
+				result = await change(tx);
+				return undefined;
+			}, commitContext);
+			return result as Awaited<ReturnType<typeof change>>;
+		},
+		memo: runtime.memo,
+		createTask: async <I, S extends { phase: string }, R, H extends object>(
+			task: Task<I, S, R, H>,
+			input: I,
+			options: Omit<TaskOptions, "conversationId">,
+			taskContext: Context,
+		): Promise<TaskId<R>> => {
+			const child = options.ownership.kind === "task" && !resumes ? { abandonOnRestart: true, ...options } : options;
+			let id: TaskId<R> | undefined;
+			await runtime.commit(async (tx) => {
+				id = await tx.createTask(task, input, child);
+				return undefined;
+			}, taskContext);
+			return id!;
+		},
+		getTask: runtime.getTask,
+		waitForTask: runtime.waitForTask,
+		conversation: runtime.conversation,
+		snapshot: runtime.snapshot,
+		snapshotAsOf: runtime.snapshotAsOf,
+		watchDoc: runtime.watchDoc,
+		executeTool: async (name, toolArgs, callContext, options = {}) => {
+			assertLive();
+			const key = options.key ?? String(++sequence);
+			if (options.key !== undefined) checkKey(options.key);
+			// The memo first, once per attempt; a failed one is tried again by the next call.
+			madeNestedMemo ??= runtime.memo(MADE_NESTED_CALLS, true, callContext).catch((error: unknown) => {
+				madeNestedMemo = undefined;
+				throw error;
+			});
+			// Registered now, so cleanup waits for it.
+			const admission = madeNestedMemo.then(() =>
+				startNestedCall(
+					runtime,
+					call.id,
+					name,
+					toolArgs,
+					{ key, progress: options.progress, abandonOnRestart: !resumes },
+					callContext,
+				),
+			);
+			admissions.push(admission);
+			const id = await admission;
+			const settled = await runtime.waitForTask(id, callContext);
+			const stored = (await runtime.snapshot(NestedCallDoc, runtime.taskId, key, callContext))?.result;
+			// A copy the tool may change; a nested call the scheduler faulted or orphaned stored none.
+			return stored === undefined
+				? fallbackResult(id, name, settled.state.outcome)
+				: (copyJson(stored as unknown as JsonValue) as unknown as NestedToolExecutionResult);
+		},
+	};
+
+	let result: ToolExecutionResult;
+	let ending = COMPLETED;
+	// Execution time of this attempt; a rerun after recovery measures only itself.
+	let durationMs: number | undefined;
+	try {
+		// Built for this call, so a rerun after recovery gets the conversation's environment at that time.
+		const env = await runtime.env(context);
+		const startedAt = performance.now();
+		try {
+			result = await tool.execute(args, { ...api, env }, context);
+		} finally {
+			durationMs = Math.round(performance.now() - startedAt);
+		}
+	} catch (error) {
+		if (runtime.signal.aborted) {
+			ended = true;
+			for (const waiter of await progress.stop()) waiter.reject(error);
+			throw error;
+		}
+		result = { isError: true, diagnostics: [toolDiagnostic("tool_error", errorMessage(error))] };
+		// A throw, from `execute()` or from building the environment, ends the task `failed`, which cancels what the call
+		// owned; it no longer supervises it. The error text is already in the result.
+		ending = { status: "failed", message: `Tool ${call.name} threw` };
+	}
+	ended = true;
+	// No admission starts after `ended`; let those underway commit, so settlement aborts and lists their calls.
+	await Promise.allSettled(admissions);
+	reported.output.end();
+	// Details still waiting for a progress commit settle with the terminal commit, the final flush.
+	const pending = await progress.stop();
+	try {
+		const settled = await finalResult(runtime, input, call, tool, result, reported, context);
+		await settle(runtime, input, call, ending, () => settled, context, durationMs);
+	} catch (error) {
+		for (const waiter of pending) waiter.reject(error);
+		throw error;
+	}
+	for (const waiter of pending) waiter.resolve();
+}
+
+/**
+ * Throttled commits of what the tool reported into its slot, in `pi.live.tools` or `pi.live.nestedTools`, each writing
+ * only what changed since the last one; none when `enabled` is false.
+ */
+function publishProgress(runtime: Runtime, reported: Reported, enabled: boolean, context: Context): Progress {
+	let written = { text: "", details: undefined as JsonValue | undefined, diagnostics: 0 };
+	return new Progress(
+		async () => {
+			if (!enabled) return 0;
+			// Capture everything synchronously: the tool keeps reporting while the commit is in flight.
+			const snapshot = reported.output.snapshot();
+			const current = { text: snapshot.text, details: reported.details, diagnostics: reported.diagnostics.length };
+			const added = reported.diagnostics.slice(written.diagnostics, current.diagnostics);
+			const detailsChanged = current.details !== written.details;
+			// What the commit writes, as Chord diffs the string: an append, a trim plus an append of what follows the shared
+			// part, or the whole window when its bounded overlap search finds nothing.
+			let bytes = 0;
+			if (snapshot.text !== written.text) {
+				const shared = snapshot.text.startsWith(written.text)
+					? written.text.length
+					: overlap(written.text, snapshot.text, 65_536);
+				bytes += utf8ByteLength(snapshot.text.slice(shared));
+			}
+			if (detailsChanged) bytes += utf8ByteLength(JSON.stringify(current.details ?? null));
+			if (added.length > 0) bytes += utf8ByteLength(JSON.stringify(added));
+			await runtime.commit(async (tx) => {
+				const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId);
+				if (slot === undefined) return undefined;
+				// REMINDER: assign `output` as one string field. Chord then diffs it into an append, or a trim plus an
+				// append for a sliding tail; replacing the slot object would record the whole window on every commit.
+				if ((slot.output ?? "") !== snapshot.text) slot.output = snapshot.text;
+				if (snapshot.droppedBytes > 0) slot.droppedBytes = snapshot.droppedBytes;
+				if (snapshot.droppedLines > 0) slot.droppedLines = snapshot.droppedLines;
+				// Diff details leaf by leaf and append new diagnostics, so each commit writes only what changed.
+				if (detailsChanged && current.details !== undefined) {
+					assignJson(slot as unknown as Record<string, JsonValue>, "details", current.details);
+				}
+				if (added.length > 0) {
+					if (slot.diagnostics === undefined) slot.diagnostics = [];
+					for (const diagnostic of added) slot.diagnostics.push(diagnostic);
+				}
+				return undefined;
+			}, context);
+			written = current;
+			return bytes;
+		},
+		(error) => {
+			// Rejections after an abort mark or close are expected; the committed state stays consistent.
+			if (!runtime.signal.aborted) runtime.report(error);
+		},
+		runtime.settings.progress.outputIntervalMs,
+	);
+}
+
+/**
+ * The settled result: the tool's result with the retained output and last details as fallbacks, its diagnostics after
+ * those reported through the api, `afterTool` applied, explicit text bounded, and `structuredOutput` checked against the
+ * tool's schema, with the Harness's diagnostics last. A nested call whose `structuredOutput` breaks the contract gets an
+ * error result without it; a model-issued call only loses it, and the break is reported to the host.
+ */
+async function finalResult(
+	runtime: Runtime,
+	input: ToolTaskInput,
+	call: ToolHookCall,
+	tool: ToolRegistration,
+	result: ToolExecutionResult,
+	reported: Reported,
+	context: Context,
+): Promise<ToolExecutionResult> {
+	const harness: ToolDiagnostic[] = [];
+	const retained = result.output === undefined ? reported.output.snapshot() : undefined;
+	const output: Content =
+		retained === undefined ? result.output! : retained.text === "" ? [] : [{ type: "text", text: retained.text }];
+	let final: ToolExecutionResult = {
+		...result,
+		output,
+		details: result.details === undefined ? reported.details : result.details,
+		diagnostics: [...reported.diagnostics, ...(result.diagnostics ?? [])],
+	};
+	await runtime.hooks.each("afterTool", async (hook) => {
+		final = (await hook(call, final, runtime, context)) ?? final;
+	});
+	// The retained output's truncation applies only while afterTool kept that output.
+	if (retained !== undefined && retained.droppedBytes > 0 && final.output === output) {
+		harness.push(truncated(retained, reported.limits.retain));
+	}
+	const bounded = boundContent(final.output ?? [], reported.limits);
+	if (bounded.droppedBytes > 0) harness.push(truncated(bounded, reported.limits.retain));
+	const broken = structuredOutputError(tool, final);
+	if (broken !== undefined) {
+		const { structuredOutput: _dropped, ...rest } = final;
+		if (input.kind === "nested") {
+			final = { ...rest, isError: true };
+			harness.push(toolDiagnostic("invalid_structured_output", broken));
+		} else {
+			// The model never sees structured output, so its call stands; the host learns of the broken tool.
+			final = rest;
+			runtime.report(new Error(broken));
+		}
+	}
+	// Programs get the tool's own value: without a schema, its output. The model reads only `output`, so model-issued
+	// calls skip both.
+	if (input.kind === "nested" && tool.structuredOutputSchema === undefined) {
+		final = { ...final, structuredOutput: outputValue(bounded.content) as JsonValue };
+	}
+	// A schema tool's error result without structured output has its text only in the output, which programs never
+	// get: it becomes an error diagnostic, as a thrown error's message does, unless one says what went wrong already.
+	if (
+		input.kind === "nested" &&
+		tool.structuredOutputSchema !== undefined &&
+		final.isError === true &&
+		final.structuredOutput === undefined &&
+		![...(final.diagnostics ?? []), ...harness].some((diagnostic) => diagnostic.severity === "error")
+	) {
+		const text = bounded.content.flatMap((item) => (item.type === "text" ? [item.text] : [])).join("\n");
+		if (text !== "") harness.push(toolDiagnostic("tool_error", text));
+	}
+	return { ...final, output: bounded.content, diagnostics: [...(final.diagnostics ?? []), ...harness] };
+}
+
+const structuredValidators = new WeakMap<object, ReturnType<typeof Compile>>();
+
+/** Why a result's `structuredOutput` breaks the tool's contract, or `undefined` when it keeps it. */
+function structuredOutputError(tool: ToolRegistration, result: ToolExecutionResult): string | undefined {
+	const schema = tool.structuredOutputSchema;
+	const value = result.structuredOutput;
+	if (schema === undefined) {
+		return value === undefined
+			? undefined
+			: `Tool ${tool.name} returned structuredOutput but declares no structuredOutputSchema`;
+	}
+	if (value === undefined) {
+		return result.isError === true ? undefined : `Tool ${tool.name} returned no structuredOutput`;
+	}
+	let validator = structuredValidators.get(schema);
+	if (validator === undefined) {
+		validator = Compile(schema);
+		structuredValidators.set(schema, validator);
+	}
+	if (validator.Check(value)) return undefined;
+	const [first] = validator.Errors(value);
+	const path = first?.instancePath.replace(/^\//, "").replace(/\//g, ".") || "root";
+	return `Tool ${tool.name} returned structuredOutput that does not match its schema: ${path}: ${first?.message ?? "invalid"}`;
+}
+
+/**
+ * Commit the tool's terminal state and mark its slot done. A model-issued call appends its result entry and ends with
+ * the entry ID; a nested call writes its result to the caller's `NestedCallDoc`, records its usage, and ends with a
+ * small receipt. Nested calls this call left running are aborted first, so they report into their slots before the
+ * slots below the call leave `pi.live`. `build` receives the slot so interruption and abort can report the durable
+ * partial output.
+ */
+async function settle(
+	runtime: Runtime,
+	input: ToolTaskInput,
+	call: ToolHookCall,
+	ending: Ending,
+	build: (slot: Readonly<ToolSlot | NestedToolSlot> | undefined) => ToolExecutionResult,
+	context: Context,
+	durationMs?: number,
+): Promise<void> {
+	// A crash in between leaves the call in `execute`: a safe rerun reattaches to the calls, an interruption ends here.
+	// Every live nested call the task owns, also those an earlier attempt made and this one never called again.
+	const nestedIds = (await runtime.ownedTasks(context))
+		.filter((record) => record.kind === ToolTask.definition.name && (record.input as ToolTaskInput).kind === "nested")
+		.map((record) => record.id);
+	// Read before the settling commit: no admission starts after `ended`, and those underway were awaited.
+	const madeNested = (await runtime.memo(MADE_NESTED_CALLS, context)) === true;
+	// Mark them all before waiting for any: one may run until a sibling is cancelled.
+	await Promise.all(nestedIds.map((id) => runtime.abortOwned(id, context)));
+	await runtime.commit(async (tx) => {
+		const conversationId = runtime.conversationId;
+		const live = await tx.doc(LiveDoc, conversationId);
+		const slot = toolSlot(live, runtime.taskId);
+		const result = build(slot);
+		let settled: ToolTaskResult;
+		if (input.kind === "nested") {
+			const nested = nestedResult(runtime.taskId, result, durationMs);
+			if (nested.usage !== undefined) await recordUsage(tx, conversationId, "tools", call.name, nested.usage);
+			// The caller is never terminal before its owned nested calls, so its document is open; another task in it is a
+			// bug, not a race.
+			const state = await tx.doc(NestedCallDoc, input.parent, input.key, null);
+			if (state.taskId !== runtime.taskId) throw new Error(`Nested call ${call.id} is not in its caller's index`);
+			state.result = nested as unknown as Draft<JsonRepresentation<NestedToolExecutionResult>>;
+			if (slot !== undefined && "parentTaskId" in slot) {
+				finishSlot(slot);
+				slot.summary = summaryOf(nested, result.output ?? []);
+			}
+			settled = { kind: "nested" };
+		} else {
+			const timing = { timestamp: runtime.now(), durationMs };
+			const entry = await appendToolResult(tx, conversationId, call, result, timing);
+			if (slot !== undefined && !("parentTaskId" in slot)) {
+				finishSlot(slot);
+				slot.entry = entry.id;
+			}
+			// Tools build control objects freely; drop keys set to undefined so the task result is strict JSON.
+			const control =
+				result.control === undefined || ending.status !== "completed"
+					? {}
+					: { control: copyJson(result.control as JsonValue, { omitUndefinedProperties: true }) as ToolControl };
+			settled = { kind: "model", entryId: entry.id, ...control };
+		}
+		// A call that made no nested calls, in any attempt, has no slots below it; skipping the scan keeps many leaves
+		// linear. Committed before the read: no nested call is admitted after `ended`.
+		if (madeNested) removeNestedSlots(live, runtime.taskId);
+		if (ending.status === "aborted") return { status: "terminal", outcome: { status: "aborted", result: settled } };
+		if (ending.status === "failed") {
+			const error = { message: ending.message };
+			return { status: "terminal", outcome: { status: "failed", error, result: settled } };
+		}
+		return { status: "terminal", outcome: { status: "completed", result: settled } };
+	}, context);
+}
+
+/**
+ * A nested call's result, as stored and returned: strict JSON, without `output`, which only the model reads, and
+ * without `control`, which only model-issued calls apply.
+ */
+function nestedResult(
+	taskId: TaskId,
+	result: ToolExecutionResult,
+	durationMs: number | undefined,
+): NestedToolExecutionResult {
+	const nested = {
+		taskId,
+		structuredOutput: result.structuredOutput,
+		isError: result.isError ?? false,
+		details: result.details,
+		diagnostics: result.diagnostics ?? [],
+		usage: result.usage,
+		durationMs,
+	};
+	return copyJson(nested as unknown as JsonValue, {
+		omitUndefinedProperties: true,
+	}) as unknown as NestedToolExecutionResult;
+}
+
+/**
+ * Create nested call `key` of the call `parentCallId` in one commit: its tool task, owned by the calling task, its slot,
+ * and its `NestedCallDoc`. A key already made returns that call's task when it names the same tool and arguments.
+ */
+async function startNestedCall(
+	runtime: Runtime,
+	parentCallId: string,
+	name: string,
+	args: JsonObject,
+	options: { readonly key: string; readonly progress: boolean | undefined; readonly abandonOnRestart: boolean },
+	context: Context,
+): Promise<TaskId<ToolTaskResult>> {
+	const key = options.key;
+	const call: ToolCall = {
+		type: "toolCall",
+		id: `${parentCallId}/${key}`,
+		name,
+		arguments: copyJson(args, { omitUndefinedProperties: true }) as JsonObject,
+	};
+	let id: TaskId<ToolTaskResult> | undefined;
+	await runtime.commit(async (tx) => {
+		// Opened, or created empty for a key not made yet.
+		const state = await tx.doc(NestedCallDoc, runtime.taskId, key, null);
+		const existing = state.taskId;
+		if (existing !== undefined) {
+			const input = (await tx.task(existing))?.input as ToolTaskInput | undefined;
+			const same =
+				input?.kind === "nested" &&
+				input.call.name === name &&
+				jsonEqual(input.call.arguments as JsonValue, call.arguments as JsonValue);
+			if (!same) throw new Error(`Nested call ${call.id} was already made with another tool or other arguments`);
+			id = existing as TaskId<ToolTaskResult>;
+			return undefined;
+		}
+		const input: ToolTaskInput = {
+			kind: "nested",
+			parent: runtime.taskId,
+			parentCallId,
+			key,
+			call,
+			...(options.progress === false ? { progress: false } : {}),
+		};
+		const created = await tx.createTask(ToolTask, input, {
+			ownership: { kind: "task", taskId: runtime.taskId },
+			abandonOnRestart: options.abandonOnRestart,
+		});
+		state.taskId = created;
+		const live = await tx.doc(LiveDoc, runtime.conversationId);
+		live.nestedTools ??= [];
+		const slot: NestedToolSlot = {
+			callId: call.id,
+			parentCallId,
+			parentTaskId: runtime.taskId,
+			name,
+			taskId: created,
+			arguments: call.arguments as JsonObject,
+			status: "pending",
+		};
+		live.nestedTools.push(slot as Draft<NestedToolSlot>);
+		id = created;
+		return undefined;
+	}, context);
+	return id!;
+}
+
+/**
+ * What programs get of a tool's output when it declares no schema: one text item as its string, one image as itself,
+ * nothing as `""`, and anything else as the content list.
+ */
+function outputValue(output: Content): string | ImageContent | Content {
+	if (output.length === 0) return "";
+	if (output.length > 1) return output;
+	const [only] = output;
+	return only!.type === "text" ? only!.text : only!;
+}
+
+/** What the caller gets for a nested call without a stored result: one the scheduler faulted or orphaned. */
+function fallbackResult(taskId: TaskId, name: string, outcome: TaskOutcome<ToolTaskResult>): NestedToolExecutionResult {
+	const message =
+		outcome.status === "faulted"
+			? `Tool ${name} failed: ${outcome.error.message}`
+			: outcome.status === "orphaned"
+				? `Tool ${name} could not resume: ${outcome.reason}`
+				: `Tool ${name} ended without a result`;
+	const error = harnessError(outcome.status, message);
+	return { taskId, isError: true, diagnostics: error.diagnostics ?? [] };
+}
+
+/**
+ * How a tool task ends; the result entry is appended either way. `failed` (execution threw or was interrupted)
+ * records cancellation intent for the conversations the call owns; a result with `isError` still completes.
+ */
+type Ending = { readonly status: "completed" | "aborted" } | { readonly status: "failed"; readonly message: string };
+
+const COMPLETED: Ending = { status: "completed" };
+
+/** An error result from the slot's durable partial output, details, and diagnostics. */
+function fromSlot(
+	slot: Readonly<ToolSlot | NestedToolSlot> | undefined,
+	code: string,
+	message: string,
+): ToolExecutionResult {
+	const diagnostics = [...(slot?.diagnostics ?? [])];
+	const droppedBytes = slot?.droppedBytes ?? 0;
+	if (droppedBytes > 0) diagnostics.push(truncated({ droppedBytes, droppedLines: slot?.droppedLines ?? 0 }));
+	diagnostics.push(toolDiagnostic(code, message));
+	return {
+		output: slot?.output === undefined || slot.output === "" ? [] : [{ type: "text", text: slot.output }],
+		isError: true,
+		...(slot?.details === undefined ? {} : { details: slot.details }),
+		diagnostics,
+	};
+}
+
+/** An error result the Harness writes itself: no content and one `error` diagnostic with `code`. */
+export function harnessError(code: string, message: string): ToolExecutionResult {
+	return { output: [], isError: true, diagnostics: [toolDiagnostic(code, message)] };
+}
+
+function toolDiagnostic(code: string, message: string): ToolDiagnostic {
+	return { severity: "error", code, message };
+}
+
+/** The Harness's truncation diagnostic; `retain` is unknown when rebuilt from a slot after recovery. */
+function truncated(
+	dropped: { readonly droppedLines: number; readonly droppedBytes: number },
+	retain?: "head" | "tail",
+): ToolDiagnostic {
+	const kept = retain === undefined ? "" : ` to its ${retain === "head" ? "beginning" : "end"}`;
+	return {
+		severity: "warn",
+		code: "truncated",
+		message: `Output truncated${kept}: ${dropped.droppedLines} lines, ${dropped.droppedBytes} bytes dropped`,
+	};
+}
+
+/** Error text a nested call's summary keeps. */
+const MAX_SUMMARY_ERROR_CHARS = 500;
+
+/** How a nested call ended, for its slot: error text, from its diagnostics or else its `output`, bounded. */
+function summaryOf(result: NestedToolExecutionResult, output: Content): NestedToolSummary {
+	const error = result.isError ? errorTextOf(result, output).slice(0, MAX_SUMMARY_ERROR_CHARS) : "";
+	return {
+		isError: result.isError,
+		...(result.durationMs === undefined ? {} : { durationMs: result.durationMs }),
+		...(result.usage === undefined ? {} : { usage: result.usage }),
+		...(error === "" ? {} : { error }),
+	};
+}
+
+/** The error messages of a nested result, else the text items of its output, joined with newlines. */
+function errorTextOf(result: NestedToolExecutionResult, output: Content): string {
+	const errors = result.diagnostics.filter((diagnostic) => diagnostic.severity === "error");
+	if (errors.length > 0) return errors.map((diagnostic) => diagnostic.message).join("\n");
+	return output.flatMap((item) => (item.type === "text" ? [item.text] : [])).join("\n");
+}
+
+/**
+ * Append a `pi.tool-result` entry. The content ends with the rendered diagnostics, so the stored message is exactly
+ * what the model sees; `data` keeps the structured list. A result's usage is added to `pi.usage` in the same commit.
+ */
+export async function appendToolResult(
+	tx: Tx,
+	conversationId: ConversationId,
+	call: ToolCall,
+	result: ToolExecutionResult,
+	meta: { readonly timestamp: number; readonly durationMs?: number },
+): Promise<TypedEntry<{ diagnostics: ToolDiagnostic[] }>> {
+	const { timestamp, durationMs } = meta;
+	const diagnostics = [...(result.diagnostics ?? [])];
+	const content: Content = [...(result.output ?? [])];
+	if (diagnostics.length > 0) content.push({ type: "text", text: renderDiagnostics(diagnostics) });
+	const message = {
+		role: "toolResult",
+		toolCallId: call.id,
+		toolName: call.name,
+		content,
+		...(result.details === undefined ? {} : { details: result.details }),
+		...(result.usage === undefined ? {} : { usage: result.usage }),
+		isError: result.isError ?? false,
+		...(durationMs === undefined ? {} : { durationMs }),
+		timestamp,
+	} as ToolResultMessage;
+	if (result.usage !== undefined) await recordUsage(tx, conversationId, "tools", call.name, result.usage);
+	return tx.appendEntry(ToolResultEntry, conversationId, { model: [message], data: { diagnostics } });
+}
+
+function renderDiagnostics(diagnostics: readonly ToolDiagnostic[]): string {
+	return `<harness>\n${diagnostics.map((diagnostic) => `[${diagnostic.severity}] ${diagnostic.message}`).join("\n")}\n</harness>`;
+}
+
+/**
+ * Bound the text of result content. When the joined text exceeds the limits, the text items are replaced by one bounded
+ * item at the position of the first (head) or last (tail) text item; other content is kept.
+ */
+function boundContent(
+	content: Content,
+	limits: OutputLimits,
+): { content: Content; droppedBytes: number; droppedLines: number } {
+	const texts = content.filter((item): item is TextContent => item.type === "text");
+	const bounded = boundOutput(texts.map((item) => item.text).join(""), limits);
+	if (bounded.droppedBytes === 0) return { content, droppedBytes: 0, droppedLines: 0 };
+	const keep = limits.retain === "head" ? texts[0] : texts.at(-1);
+	const result: Content = [];
+	for (const item of content) {
+		if (item.type !== "text") result.push(item);
+		else if (item === keep) result.push({ ...item, text: bounded.text });
+	}
+	return { content: result, droppedBytes: bounded.droppedBytes, droppedLines: bounded.droppedLines };
+}
+
+/**
+ * Explicit nested call keys are path segments of the call ID, so IDs of different nested calls never collide, and are
+ * never plain positive integers, which default keys use. `__proto__` stays reserved, as it was when keys indexed a JSON object.
+ */
+function checkKey(key: string): void {
+	if (key === "" || key.includes("/") || key === "__proto__" || /^[1-9][0-9]*$/.test(key)) {
+		throw new Error(
+			`Nested call key ${JSON.stringify(key)} must be non-empty, without "/", not "__proto__", and not a positive integer`,
+		);
+	}
+}
+
+/** Structural JSON equality; object key order does not matter. */
+function jsonEqual(a: JsonValue, b: JsonValue): boolean {
+	if (a === b) return true;
+	if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) return false;
+	if (Array.isArray(a) || Array.isArray(b)) {
+		return (
+			Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, i) => jsonEqual(item, b[i]!))
+		);
+	}
+	const keys = Object.keys(a);
+	return (
+		keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && jsonEqual(a[key]!, b[key]!))
+	);
+}

third_party/pi/packages/durable/src/harness/types.ts

@@ -0,0 +1,812 @@
+import type { AttachedReplicatedState, Context, JsonValue } from "@earendil-works/chord";
+import type {
+	AssistantMessage,
+	CacheRetention,
+	Message,
+	Models,
+	ModelThinkingLevel,
+	Static,
+	Tool,
+	ToolCall,
+	ToolResultMessage,
+	Transport,
+	TSchema,
+	Usage,
+	UserMessage,
+} from "@earendil-works/pi-ai";
+import type { ExecutionEnv, ShellOutputSkip, ShellOutputWindow } from "../env/index.ts";
+import type {
+	ConversationId,
+	ConversationOwnership,
+	ConversationRecord,
+	Cursor,
+	DocumentObserver,
+	DocumentReader,
+	EntryDraft,
+	EntryId,
+	EntryQuery,
+	EntryRecord,
+	JsonObject,
+	Page,
+	Session,
+	SubmissionId,
+	SubmissionRecord,
+	Task,
+	TaskId,
+	TaskOptions,
+	TaskRecord,
+	TaskState,
+	Tx,
+	WatchHandle,
+} from "../types.ts";
+import type { TaskGraph, TaskGraphWatch } from "./task-graph.ts";
+import type { UsageState } from "./usage.ts";
+import type { ConversationView } from "./view.ts";
+
+/** Provider and model ID resolved through pi-ai `Models`. */
+export type ModelRef = {
+	readonly provider: string;
+	readonly modelId: string;
+};
+
+export type UserInput = UserMessage["content"];
+
+/**
+ * The entry an input is placed as, instead of `pi.user`: an application kind, and data the transcript shows, such as its
+ * author. Its model message is still the input's user message.
+ */
+export type InputEntry = { readonly kind: string; readonly data?: JsonValue };
+
+/** Host submission: user input that may start a run, or a passive entry write. */
+export type SubmissionDraft = {
+	readonly requestId?: string;
+} & (
+	| {
+			readonly type: "input";
+			readonly content: UserInput;
+			readonly whenBusy?: "steer" | "followUp" | "reject";
+			/** Place the input as this entry; default `pi.user` without data. The kind must not start with `pi.`. */
+			readonly entry?: InputEntry;
+	  }
+	| {
+			readonly type: "write";
+			readonly entry: EntryDraft;
+			readonly content?: never;
+			readonly whenBusy?: never;
+	  }
+);
+
+export type InputSubmissionDraft = Extract<SubmissionDraft, { readonly type: "input" }>;
+
+export type SettledSubmissionRecord = SubmissionRecord & {
+	readonly status: "done" | "unanswered";
+};
+
+/** Awaitable host object for one durably admitted submission. */
+export interface Submission {
+	readonly id: SubmissionId;
+	status(context: Context): Promise<SubmissionRecord>;
+	wait(context: Context): Promise<SettledSubmissionRecord>;
+	abort(context: Context): Promise<"aborted" | "already_placed" | "settled">;
+}
+
+export type SettledTask<R> = TaskRecord<JsonValue, JsonValue, R> & {
+	readonly state: Extract<TaskState<JsonValue, R>, { readonly status: "terminal" }>;
+};
+
+/** Erased executable task definition stored in the registry. */
+export type AnyTask = {
+	readonly definition: {
+		readonly name: string;
+		readonly version: number;
+		readonly initial: unknown;
+		readonly phases: Readonly<Record<string, unknown>>;
+		readonly abort: unknown;
+		readonly migrate?: unknown;
+		readonly hooks?: object;
+	};
+};
+
+/** Options of `Conversation.abort()`. */
+export type ConversationAbortOptions = {
+	/**
+	 * Cross background boundaries: mark every live task reached ignoring the background flag when the abort is admitted,
+	 * withdraw the queued inputs of every conversation reached, and wait until those tasks are terminal and the
+	 * conversation is ordinarily idle. Background work created afterwards is neither marked nor awaited.
+	 */
+	readonly background?: boolean;
+};
+
+/** Hook handler map declared by a task definition. */
+export type HooksOf<K> = K extends Task<infer _I, infer _S, infer _R, infer H> ? H : never;
+
+/**
+ * Invocation-bound conversation operations for tasks and tools. Rejects after the invocation ends; passive entries are
+ * written with ordinary transaction writes instead.
+ */
+export interface ConversationHandle {
+	readonly id: ConversationId;
+	submit(submission: InputSubmissionDraft, context: Context): Promise<Submission>;
+	/** `Conversation.abort()`: withdraw queued inputs, abort the ordinary ownership scope, and wait until it is idle. */
+	abort(context: Context, options?: ConversationAbortOptions): Promise<void>;
+	/** Resolve when the conversation's ordinary ownership scope has no live non-background task. */
+	waitForIdle(context: Context): Promise<void>;
+}
+
+/** Post-tools controls requested by a tool result. */
+export type ToolControl = {
+	readonly addTools?: readonly string[];
+	readonly terminate?: true;
+	readonly handoff?: string;
+};
+
+/** Remark about a call for the model and the UI, such as truncation or a spill path; never part of the tool's data. */
+export type ToolDiagnostic = {
+	readonly severity: "info" | "warn" | "error";
+	readonly message: string;
+	readonly code?: string;
+};
+
+/**
+ * What a tool returns, on three channels: `output` for the model, `structuredOutput` for programs that call the tool
+ * (`executeTool()` callers, code mode scripts), and `details` for UIs.
+ */
+export type ToolExecutionResult<TDetails extends JsonValue = JsonValue> = {
+	/** For the model. Omitted: the retained `output()` text. */
+	readonly output?: ToolResultMessage["content"];
+	/**
+	 * For programs. Required unless `isError` when the tool declares `structuredOutputSchema`, and validated against it;
+	 * not allowed without one, where programs get the output itself (`NestedToolExecutionResult.structuredOutput`).
+	 */
+	readonly structuredOutput?: JsonValue;
+	readonly isError?: boolean;
+	/** Omitted: the last `details()` value becomes the details. */
+	readonly details?: TDetails;
+	/** Added after those recorded through `api.diagnostic()`. */
+	readonly diagnostics?: readonly ToolDiagnostic[];
+	/** Spend of the execution itself, such as a model call; stored on the result and in `pi.usage.tools`. */
+	readonly usage?: Usage;
+	readonly control?: ToolControl;
+};
+
+/**
+ * Result of a nested call, as `executeTool()` returns it. Programs read `structuredOutput`; the output for the model
+ * is not kept, and `control` does not apply.
+ */
+export type NestedToolExecutionResult = {
+	/** The nested call's tool task. */
+	readonly taskId: TaskId;
+	/**
+	 * The tool's value for programs. A tool with `structuredOutputSchema`: its validated `structuredOutput`, also on an
+	 * error result that has one; absent otherwise, and an error's text is then in an error diagnostic. A tool without:
+	 * its bounded output, success or error, as a string for one text item, an `ImageContent` for one image, `""` for
+	 * none, and the content list otherwise. Absent on an error result the Harness wrote itself (unavailable, invalid,
+	 * blocked, interrupted, aborted); `diagnostics` say what went wrong.
+	 */
+	readonly structuredOutput?: JsonValue;
+	readonly isError: boolean;
+	readonly details?: JsonValue;
+	readonly diagnostics: readonly ToolDiagnostic[];
+	readonly usage?: Usage;
+	/** Execution time of the attempt that produced the result; absent when it never executed. */
+	readonly durationMs?: number;
+};
+
+/** Whether the tools of one round run at once or one after another in call order. */
+export type ToolExecutionMode = "parallel" | "sequential";
+
+/** How many queued items of one mode a boundary places: the first, or all of them. */
+export type QueueMode = "all" | "one-at-a-time";
+
+/**
+ * Operations available to one tool invocation. A plain object, so a wrapper can spread it. Every operation rejects after
+ * the invocation ends.
+ */
+export interface ToolExecutionApi<TDetails extends JsonValue = JsonValue> extends DocumentObserver, DocumentReader {
+	readonly taskId: TaskId;
+	readonly conversationId: ConversationId;
+	readonly callId: string;
+	/** The tool task's phase snapshot. */
+	readonly registry: RegistrySnapshot;
+	/** The calling conversation's agent, as the tool task's phase resolved it. */
+	agent(context: Context): Promise<Agent>;
+	/** `HarnessOptions.models`: the catalog, credentials, and request transforms generation uses. */
+	readonly models: Models;
+	/** Built by `HarnessOptions.env` for this call; `undefined` without an environment. */
+	readonly env: ExecutionEnv | undefined;
+	/**
+	 * Append running output; it becomes the result's `output` when the result omits it. `skipped` counts output
+	 * omitted before the chunk, as reported by an environment given `outputWindow` (`ShellOutputInfo.skipped`).
+	 */
+	output(chunk: string | Uint8Array, skipped?: ShellOutputSkip): void;
+	/**
+	 * The tail this call's output keeps and the pace of its progress commits, for `ShellExecOptions.window`; `undefined`
+	 * when the tool keeps the head of its output, which cannot accept skips. A wrapper that replaces `output` and
+	 * transforms text must also replace this with `undefined`, so skipped text cannot bypass its transform.
+	 */
+	readonly outputWindow: ShellOutputWindow | undefined;
+	/**
+	 * The output retained so far, as the model will see it when the result omits `output`, and whether earlier output was
+	 * dropped; for a tool that also returns that text in its `structuredOutput`.
+	 */
+	retainedOutput(): { readonly text: string; readonly truncated: boolean };
+	/** Record a model-visible remark about this call. */
+	diagnostic(diagnostic: ToolDiagnostic): void;
+	/** Replace running details; the last value becomes the result details when the result omits `details`. */
+	details(value: TDetails, context: Context): Promise<void>;
+	commit<T>(change: (tx: Tx) => T | Promise<T>, context: Context): Promise<T>;
+	memo<T extends JsonValue>(name: string, context: Context): Promise<T | undefined>;
+	memo<T extends JsonValue>(name: string, candidate: T, context: Context): Promise<T>;
+	/**
+	 * Create a task in one commit. A child task of a tool that is not replay-safe defaults to `abandonOnRestart`: the
+	 * tool never resumes after a restart, so nothing awaits the child then.
+	 */
+	createTask<I, S extends { phase: string }, R, H extends object>(
+		task: Task<I, S, R, H>,
+		input: I,
+		options: Omit<TaskOptions, "conversationId">,
+		context: Context,
+	): Promise<TaskId<R>>;
+	getTask<R>(id: TaskId<R>, context: Context): Promise<TaskRecord<JsonValue, JsonValue, R> | undefined>;
+	waitForTask<R>(id: TaskId<R>, context: Context): Promise<SettledTask<R>>;
+	/** Invocation-bound handle of an existing conversation, such as one this tool created in `commit()`. */
+	conversation(id: ConversationId, context: Context): Promise<ConversationHandle | undefined>;
+	/**
+	 * Run tool `name` as a nested call of this one and wait for its result. The nested call is its own `pi.tool` task,
+	 * owned by this call: it resolves the tool among the conversation's tools, validates, runs the `ToolTask` hooks (which
+	 * see `call.parent`), applies output limits and the tool's replay policy, and reports progress in `pi.live.nestedTools`.
+	 * Its result returns here instead of entering the transcript. A call that is blocked, invalid, throws, is interrupted,
+	 * or is aborted returns an `isError` result.
+	 *
+	 * The nested call's key names it within this call; its call ID is `<callId>/<key>`. By default the key is the call's
+	 * position among this invocation's nested calls, `1`, `2`, ... An explicit `key` must be non-empty, without `/`, not
+	 * `__proto__`, and not a positive integer, so it never collides with a default one. A rerun of a replay-safe tool
+	 * that makes a call with a used key, by calling in the same order or by passing the same `key`, gets that nested call
+	 * back, finished or still running, instead of starting another. A nested call a crash interrupted before its result
+	 * committed follows its own replay policy, so this is not exactly-once execution. Reusing a key with another tool or
+	 * other arguments throws. A tool that is not replay-safe never reruns: after a restart, its unfinished nested calls
+	 * are abandoned (`TaskOptions.abandonOnRestart`) before they run again.
+	 *
+	 * `progress: false` commits no running output, details, or diagnostics to the nested call's slot, only its status;
+	 * the result still carries its details and diagnostics, and a schema-less tool's output as `structuredOutput`. An
+	 * interrupted or aborted call's result is built from its slot, so it then has none of them. Either way, programs never
+	 * get the partial output of an interrupted or aborted call.
+	 *
+	 * Nested calls run as soon as they are made, whatever their `executionMode`; the caller decides what runs at once by
+	 * what it awaits. Cancelling `context` stops only the wait; the nested call runs on. When this call settles, nested
+	 * calls it left running are aborted first. A nested result's `usage` is already counted under its tool; a result
+	 * that passes it on counts it again. Rejects, instead of returning a result, once `execute()` has returned, when this
+	 * call is aborted, or when the Harness closes.
+	 */
+	executeTool(
+		name: string,
+		args: JsonObject,
+		context: Context,
+		options?: { readonly key?: string; readonly progress?: boolean },
+	): Promise<NestedToolExecutionResult>;
+}
+
+/**
+ * Executable tool registered in a registry. Only pi-ai `Tool` fields enter the transcript. `args` are typed by
+ * `parameters`, which the Harness validates them against before `execute()`; `defineTool()` infers both generics.
+ */
+export type ToolRegistration<
+	TParameters extends TSchema = TSchema,
+	TDetails extends JsonValue = JsonValue,
+> = Tool<TParameters> & {
+	/** Whether an interrupted execution may rerun on recovery. Default `unsafe`. */
+	readonly replay?: "safe" | "unsafe";
+	/** Default: the settings' `toolExecution`. One sequential call makes its whole round sequential. */
+	readonly executionMode?: ToolExecutionMode;
+	/**
+	 * Repair arguments models commonly get wrong before validation, such as a JSON string where an array belongs. Must be
+	 * pure and must not mutate `args`: it runs again when a call is retried before its intent is recorded. Its result is
+	 * still validated against `parameters`.
+	 */
+	prepareArguments?(args: unknown): Static<TParameters>;
+	/**
+	 * Schema of `structuredOutput`, which programs that call the tool receive. Omitted: they receive the result's bounded
+	 * output, success or error, as `NestedToolExecutionResult.structuredOutput` describes.
+	 */
+	readonly structuredOutputSchema?: TSchema;
+	/**
+	 * Who may call the tool: the model, other tools through `executeTool()`, or both (default). A code mode tool is
+	 * `["model"]`, so scripts cannot start scripts; tools behind code mode, such as MCP tools, can be `["tools"]`, so the
+	 * model is never offered them.
+	 */
+	readonly callers?: readonly ("model" | "tools")[];
+	/**
+	 * Not offered to the model until loaded: the model gets the tool only when `modelTools` names it, in its array or its
+	 * `add`, which `control.addTools` (from a tool such as `tool_search`) writes. Other tools can call it any time.
+	 */
+	readonly deferred?: true;
+	readonly outputLimits?: {
+		readonly maxBytes?: number;
+		readonly maxLines?: number;
+		readonly retain?: "head" | "tail";
+	};
+	execute(
+		args: Static<TParameters>,
+		api: ToolExecutionApi<TDetails>,
+		context: Context,
+	): Promise<ToolExecutionResult<TDetails>>;
+};
+
+/** Input to system prompt section rendering for one request preparation. */
+export type PromptInput<Tool extends ToolRegistration = ToolRegistration> = {
+	readonly conversationId: ConversationId;
+	/** The request's resolution; `agent.tools` are the tools offered in this request. */
+	readonly agent: Agent<Tool>;
+	/** Built by `HarnessOptions.env` for this preparation; `undefined` without an environment. */
+	readonly env: ExecutionEnv | undefined;
+	/** Sections already in effect after replaying the active transcript. */
+	readonly shown: Readonly<Record<string, string>>;
+	/** Committed document reads. */
+	readonly read: DocumentReader;
+};
+
+/** One system prompt section; the agent's sections render in order before each request. */
+export type PromptSection<Tool extends ToolRegistration = ToolRegistration> = {
+	readonly key: string;
+	render(input: PromptInput<Tool>, context: Context): string | undefined | Promise<string | undefined>;
+	/** Default true: wrap the text as `<key>\n...\n</key>`. */
+	readonly tag?: boolean;
+};
+
+/** Built by `hook()`; matches tasks by name. */
+export type HookRegistration = { readonly task: string; readonly handlers: object };
+
+/** Built by `wrapTool()` and `wrapSection()`; targets a tool name or a section key. Wrappers are pure. */
+export type Wrap<Tool extends ToolRegistration = ToolRegistration> =
+	| { readonly tool: string; wrap(tool: Tool): Tool }
+	| { readonly section: string; wrap(section: PromptSection<Tool>): PromptSection<Tool> };
+
+/** Named bundle of code; installed in a registry and selected by conversations by name. */
+export interface Extension<Tool extends ToolRegistration = ToolRegistration> {
+	readonly name: string;
+	readonly tools?: readonly Tool[];
+	readonly sections?: readonly PromptSection<Tool>[];
+	readonly hooks?: readonly HookRegistration[];
+	/** Apply where this extension is selected, in order. */
+	readonly wraps?: readonly Wrap<Tool>[];
+	/** Resolved by name for every task, whichever conversations select this extension. */
+	readonly tasks?: readonly AnyTask[];
+}
+
+/** Immutable view of one published registry state. */
+export interface RegistrySnapshot<Tool extends ToolRegistration = ToolRegistration> {
+	installed(): readonly Extension<Tool>[];
+	extension(name: string): Extension<Tool> | undefined;
+	/** Every installed tool with its extension, in install order. Names may repeat across extensions. */
+	tools(): readonly { readonly extension: Extension<Tool>; readonly tool: Tool }[];
+	sections(): readonly { readonly extension: Extension<Tool>; readonly section: PromptSection<Tool> }[];
+	/** Built-in and installed task definitions. */
+	tasks(): readonly AnyTask[];
+	task(name: string): AnyTask | undefined;
+}
+
+/** Read side of a registry consumed by a Harness. */
+export interface RegistryReader<Tool extends ToolRegistration = ToolRegistration> {
+	/** Immutable view of the whole current registry. */
+	snapshot(): RegistrySnapshot<Tool>;
+	/** Called synchronously after every publication; wakes the scheduler to reconsider blocked tasks. */
+	subscribe(listener: () => void): () => void;
+}
+
+/** Application-owned registry of extensions. */
+export interface Registry<Tool extends ToolRegistration = ToolRegistration> extends RegistryReader<Tool> {
+	/** Install `extension`, or replace the installed extension with its name in place. Publishes at once. */
+	install(extension: Extension<Tool>): void;
+	/** Remove the installed extension with `extension.name`, whichever object it is. A later install appends. */
+	uninstall(extension: Extension): void;
+}
+
+/** Stored choices of one conversation; names, not objects. Unset fields follow the host. */
+export type AgentState = {
+	model?: ModelRef;
+	thinkingLevel?: ModelThinkingLevel;
+	/** An array selects exactly these extensions, in order. An object edits the host default selection. */
+	extensions?: string[] | { add?: string[]; remove?: string[] };
+	/** Filters the selected extensions' tools: the enabled tools. An array enables exactly these, in order. */
+	tools?: string[] | { remove: string[] };
+	/**
+	 * Filters the enabled tools the model may call (`callers` includes `model`): the tools offered to it. Never widens:
+	 * other tools can still call the ones it leaves out.
+	 * - Unset: every such tool except deferred ones.
+	 * - An array: exactly these, in order, deferred ones too.
+	 * - An object: every such tool except deferred ones and those in `remove`, in registry order, followed by the
+	 *   deferred tools in `add` that are not in `remove`, in `add` order. Names of tools that are not deferred or not
+	 *   registered have no effect in `add`.
+	 */
+	modelTools?: string[] | { remove?: string[]; add?: string[] };
+	/** Rendered after every extension section, as the section `instructions`. */
+	instructions?: string;
+	/** Directory within the environment's file system, passed to `HarnessOptions.env`. */
+	cwd?: string;
+};
+
+/** A change to `pi.agent`: a given field replaces the stored one, `null` clears it, `undefined` changes nothing. */
+export type AgentChange = {
+	readonly model?: ModelRef | null;
+	readonly thinkingLevel?: ModelThinkingLevel | null;
+	readonly extensions?:
+		| readonly Extension[]
+		| { readonly add?: readonly Extension[]; readonly remove?: readonly Extension[] }
+		| null;
+	readonly tools?: readonly ToolRegistration[] | { readonly remove: readonly ToolRegistration[] } | null;
+	/** Replaces the stored filter as a whole; tools loaded with `control.addTools` are in its `add`. */
+	readonly modelTools?:
+		| readonly ToolRegistration[]
+		| { readonly remove?: readonly ToolRegistration[]; readonly add?: readonly ToolRegistration[] }
+		| null;
+	readonly instructions?: string | null;
+	readonly cwd?: string | null;
+};
+
+/** A conversation's agent resolved against a registry snapshot and the settings. */
+export type Agent<Tool extends ToolRegistration = ToolRegistration> = {
+	readonly model?: ModelRef;
+	readonly thinkingLevel: ModelThinkingLevel;
+	readonly extensions: readonly Extension<Tool>[];
+	/** The tools a request offers, in order: enabled, callable by the model, and selected by `modelTools`. */
+	readonly tools: readonly Tool[];
+	/** The tools nested calls (`executeTool()`) resolve among, in order: enabled and callable by tools. */
+	readonly callable: readonly Tool[];
+	/** Enabled deferred tools the model may call and is not offered yet, in registry order: what `tool_search` loads. */
+	readonly deferred: readonly Tool[];
+	/** Extension sections, then `instructions` when set. */
+	readonly sections: readonly PromptSection<Tool>[];
+	readonly instructions?: string;
+	readonly cwd?: string;
+};
+
+/**
+ * Runs inside the creating commit, after the creation hook and the `agent` change. The conversation creation is already
+ * a table write, so table reads here throw `ReadAfterWrite`; document access remains available.
+ */
+export type ConversationInit = (tx: Tx, conversationId: ConversationId) => void | Promise<void>;
+
+export type ConversationCreateOptions = {
+	readonly ownership: ConversationOwnership;
+	/** Applied in the creating commit after the creation hook's copy, before `init`. */
+	readonly agent?: AgentChange;
+	readonly init?: ConversationInit;
+};
+
+/** Curated pi-ai request options; absent fields use pi-ai defaults. */
+export type ConversationStreamOptions = {
+	transport?: Transport;
+	timeoutMs?: number;
+	/** Provider/SDK retries inside one request attempt. */
+	maxRetries?: number;
+	maxRetryDelayMs?: number;
+	headers?: Record<string, string>;
+	metadata?: JsonObject;
+	cacheRetention?: CacheRetention;
+	deferred?: boolean | { window?: "15m" | "1h" | "24h" };
+};
+
+/** Durable generation attempt retries; the JSON shape of pi-ai `RetryPolicy`. */
+export type ConversationRetryPolicy = {
+	enabled: boolean;
+	maxRetries: number;
+	baseDelayMs: number;
+	maxAgentDelayMs?: number;
+};
+
+/** Automatic compaction thresholds (spec §8.7); manual compaction ignores `enabled`. */
+export type CompactionPolicy = {
+	/** Threshold and overflow compaction. */
+	enabled: boolean;
+	/** Room kept free for the answer: generation blocks to compact above `contextWindow - reserveTokens`. */
+	reserveTokens: number;
+	/** Approximate size of the recent context a summary keeps verbatim. */
+	keepRecentTokens: number;
+	/** Background compaction starts `backgroundTokens` below the blocking threshold; `0` disables it. */
+	backgroundTokens: number;
+};
+
+/**
+ * How often running progress is committed. Each progress commit is a storage write; a host whose storage is remote can
+ * commit less often, so live answers and tool output appear in larger steps.
+ */
+export type ProgressPolicy = {
+	/** Minimum pause between commits of the answer being generated. */
+	partialIntervalMs: number;
+	/** Minimum pause between commits of running tool output; large commits also pause in proportion to their size. */
+	outputIntervalMs: number;
+};
+
+/** Why a compaction runs: `compact()`, a threshold in generation preparation, or a context overflow. */
+export type CompactionReason = "manual" | "threshold" | "overflow";
+
+/**
+ * `entryId` of a blocking compaction's summary, or the `submissionId` of a conversation-owned compaction's summary
+ * write; both absent when nothing was compacted.
+ */
+export type CompactionResult = { entryId?: EntryId; submissionId?: SubmissionId };
+
+/** Harness-wide run policy. Read at every resolution and never copied; getters are fine. Synchronous: some readers run on the Session line. */
+export type HarnessSettings = {
+	/** Default extension selection; absent: every installed extension, in install order. */
+	readonly extensions?: readonly Extension[];
+	readonly stream?: ConversationStreamOptions;
+	readonly retry?: Partial<ConversationRetryPolicy>;
+	readonly compaction?: Partial<CompactionPolicy>;
+	readonly progress?: Partial<ProgressPolicy>;
+	readonly toolExecution?: ToolExecutionMode;
+	readonly steeringMode?: QueueMode;
+	readonly followUpMode?: QueueMode;
+	/**
+	 * How long an idle conversation keeps its last context read in memory, so its next run reads only newer entries.
+	 * Busy conversations always keep it; `0` drops it once the conversation is idle.
+	 */
+	readonly contextRetentionMs?: number;
+	/**
+	 * How long a loaded document that no watch or document state observes stays in memory after its last read or write;
+	 * the next access loads it from Storage. Observed documents always stay. Default ten minutes; `0` drops one right
+	 * after use; `Infinity` keeps them. Anything else that is not a number of milliseconds means the default.
+	 */
+	readonly documentRetentionMs?: number;
+};
+
+/** Resolved settings: every field over its built-in default, object fields merged. */
+export type Settings = {
+	/** Absent: every installed extension, in install order. */
+	readonly extensions?: readonly Extension[];
+	readonly stream: ConversationStreamOptions;
+	readonly retry: ConversationRetryPolicy;
+	readonly compaction: CompactionPolicy;
+	readonly progress: ProgressPolicy;
+	readonly toolExecution: ToolExecutionMode;
+	readonly steeringMode: QueueMode;
+	readonly followUpMode: QueueMode;
+	readonly contextRetentionMs: number;
+	readonly documentRetentionMs: number;
+};
+
+/** What `HarnessOptions.env` builds an environment for. */
+export type EnvTarget = {
+	readonly conversationId: ConversationId;
+	/** The conversation's agent `cwd`. */
+	readonly cwd?: string;
+	readonly read: DocumentReader;
+};
+
+export type HarnessOptions<Tool extends ToolRegistration = ToolRegistration> = {
+	/** pi-ai model access used by generation. */
+	readonly models: Models;
+	readonly registry: RegistryReader<Tool>;
+	readonly settings?: HarnessSettings;
+	/** Builds a conversation's environment at each use. Never called on the Session line; may be async. */
+	readonly env?: (target: EnvTarget, context: Context) => ExecutionEnv | undefined | Promise<ExecutionEnv | undefined>;
+	/**
+	 * Runs in every commit that creates or forks a conversation, raw `tx.createConversation()` included, after the
+	 * built-in `pi.*` documents and before the conveniences apply `agent` and run `init`. A fork already has its copies.
+	 * Table reads throw `ReadAfterWrite`, as in `init`; a throw fails the creating commit.
+	 */
+	readonly conversationCreated?: (tx: Tx, conversation: ConversationRecord) => void | Promise<void>;
+	readonly now?: () => number;
+	/**
+	 * Receives failures that do not fail the calling operation, such as a throwing hook or listener, and the error that
+	 * fails the Session, once. Its own throw or rejection is dropped.
+	 */
+	readonly onReport?: (error: unknown) => void;
+};
+
+/** Live task and what the scheduler would do with it under the current registry. */
+export type TaskInspection = {
+	readonly record: TaskRecord<JsonValue, JsonValue, JsonValue>;
+	readonly state: /** An invocation is active. */
+		| { readonly kind: "running" }
+		/** The next scheduling pass reserves it; `migrates` when its definition is newer and has `migrate`. */
+		| { readonly kind: "ready"; readonly migrates: boolean }
+		/**
+		 * Waits for these live tasks: the live part of its `on`, or, when abort-marked, its live ordinary owned work, which
+		 * must end before its abort handler starts.
+		 */
+		| { readonly kind: "waiting"; readonly on: readonly TaskId[] }
+		/** Outcome held until its ordinary owned work drains. */
+		| { readonly kind: "completing" }
+		/** No registered definition can take it; aborting it settles it as `orphaned`. */
+		| {
+				readonly kind: "blocked";
+				readonly reason: "missing_task" | "task_too_old" | "migration_failed";
+				readonly error?: unknown;
+		  };
+};
+
+/** Point-in-time view of live work: unfinished tasks and submissions, read on the Session line. */
+export type HarnessInspection = {
+	readonly scheduling: "paused" | "running" | "closing";
+	readonly tasks: readonly TaskInspection[];
+	/** Queued and placed submissions, in ID order. */
+	readonly submissions: readonly SubmissionRecord[];
+};
+
+/** Raw active transcript and derived model context. */
+export type ContextView = {
+	/** Newest applicable head marker, if any. */
+	readonly head: EntryRecord | undefined;
+	/** Raw active entries: the head marker followed by non-head entries from its head through the tail. */
+	readonly entries: readonly EntryRecord[];
+	/** Per entry of `entries`, its model messages after edits and excluded stop reasons, before tool result ordering. */
+	readonly contributions: readonly (readonly Message[])[];
+	/** Model context for the next provider request. */
+	readonly messages: readonly Message[];
+};
+
+/** Stateless handle for one conversation, bound to the Harness that returned it. Compare handles by `id`. */
+export interface Conversation {
+	readonly id: ConversationId;
+
+	/** Resolved with the current registry snapshot and settings. */
+	agent(context: Context): Promise<Agent>;
+	/** `configure()` in its own commit. */
+	configure(change: AgentChange, context: Context): Promise<void>;
+
+	/**
+	 * Durably admit user input or a passive entry write. A busy conversation, or one with queued items, queues input and
+	 * writes that affect model context in `pi.inbox`; `whenBusy: "reject"` rejects input with `ConversationBusy` instead
+	 * and writes nothing. A write of only `kind` and `data` (no `model`, `head`, or `edits`) is appended at once,
+	 * even mid-run, as it changes nothing the model sees.
+	 */
+	submit(submission: SubmissionDraft, context: Context): Promise<Submission>;
+	/**
+	 * Admit a write of a `pi.reset` entry that starts a new context, carrying `handoff` as a user message when given.
+	 * Resolves after admission; while busy, it is placed at the next boundary.
+	 */
+	reset(handoff: string | undefined, context: Context): Promise<void>;
+	/**
+	 * Admit a manual compaction task and return its ID. It summarizes while the conversation keeps working and places its
+	 * summary through a write submission: at once when idle, otherwise at the next boundary (spec §8.7).
+	 */
+	compact(instructions: string | undefined, context: Context): Promise<TaskId<CompactionResult>>;
+
+	/** Session commit whose `tx.createTask()` defaults to this conversation. */
+	commit<T>(change: (tx: Tx) => T | Promise<T>, context: Context): Promise<T>;
+	/**
+	 * Committed raw active transcript and model context. With `at`, the context as of that visible entry: the same view
+	 * `fork(at)` would start with, without creating a conversation.
+	 */
+	context(context: Context, options?: { readonly at?: EntryId }): Promise<ContextView>;
+	/** Fork-aware history of this conversation, newest first unless `query.order` is `ascending`. */
+	entries(
+		query: Omit<EntryQuery, "conversationId">,
+		limit: number,
+		cursor: Cursor | undefined,
+		context: Context,
+	): Promise<Page<EntryRecord, Cursor>>;
+	fork(at: EntryId, options: ConversationCreateOptions, context: Context): Promise<Conversation>;
+	/**
+	 * Withdraw queued inputs (queued writes stay), mark every live non-background task of the ordinary ownership scope,
+	 * signal them, and resolve once the scope is idle. Background subtrees survive unless `background` is set.
+	 */
+	abort(context: Context, options?: ConversationAbortOptions): Promise<void>;
+	/**
+	 * Resolve when the ordinary ownership scope has no live non-background task: this conversation and the conversations
+	 * owned, transitively, by its non-background tasks.
+	 */
+	waitForIdle(context: Context): Promise<void>;
+	/** The structural view (spec §9.3) as a disposable read-only Chord state. */
+	viewState(context: Context): Promise<AttachedReplicatedState<ConversationView>>;
+	/** The structural view as a serialized exact-frame watch with bounded pending frames. */
+	watch(context: Context): Promise<ConversationWatch>;
+}
+
+export type ConversationWatch = WatchHandle<ConversationView>;
+
+/** Durable agent harness over one Session. */
+export interface Harness extends Session {
+	/**
+	 * Enable task scheduling. Idempotent; throws after close. Calls that ask for progress enable it too:
+	 * `Conversation.submit()`, `Conversation.compact()`, `Conversation.abort()`, `Submission.wait()`, `waitForTask()`,
+	 * `Harness.waitForIdle()`, and `Conversation.waitForIdle()`. Read-only viewers never do.
+	 */
+	resume(): void;
+
+	/** Return the reserved root conversation, creating it with `agent` and `init` in one commit when absent. */
+	root(
+		context: Context,
+		options?: { readonly agent?: AgentChange; readonly init?: ConversationInit },
+	): Promise<Conversation>;
+	conversation(id: ConversationId, context: Context): Promise<Conversation | undefined>;
+	createConversation(options: ConversationCreateOptions, context: Context): Promise<Conversation>;
+
+	getTask<R>(id: TaskId<R>, context: Context): Promise<TaskRecord<JsonValue, JsonValue, R> | undefined>;
+	/** Live tasks and unsettled submissions. Writes nothing and runs no task code. */
+	inspect(context: Context): Promise<HarnessInspection>;
+	/** Reacquire a submission, for example after reopen. */
+	submission(id: SubmissionId, context: Context): Promise<Submission | undefined>;
+	/** `not_found` for an unknown submission or one of another conversation than `conversationId`. */
+	abortSubmission(
+		id: SubmissionId,
+		context: Context,
+		conversationId?: ConversationId,
+	): Promise<"aborted" | "already_placed" | "settled" | "not_found">;
+	/**
+	 * Commit the abort mark, signal and join an active run invocation, and schedule the abort invocation. A task whose
+	 * definition cannot take it settles as `orphaned` instead.
+	 */
+	abortTask(id: TaskId, context: Context): Promise<"marked" | "terminal">;
+	/** Resolve with the terminal receipt; cancelling `context` cancels only this wait. */
+	waitForTask<R>(id: TaskId<R>, context: Context): Promise<SettledTask<R>>;
+	/** Resolve when the ordinary ownership scope of every ownerless conversation has no live non-background task. */
+	waitForIdle(context: Context): Promise<void>;
+	/** Session total: every conversation's `pi.usage` summed. */
+	usage(context: Context): Promise<UsageState>;
+	/** Every live task with its owner edge, status, and owned conversations (spec §9.5), as a disposable Chord state. */
+	taskGraph(context: Context): Promise<AttachedReplicatedState<TaskGraph>>;
+	/** The task graph as a serialized exact-frame watch with bounded pending frames. */
+	watchTaskGraph(context: Context): Promise<TaskGraphWatch>;
+}
+
+/** What a hook may use: committed reads and the asking task's memos, which hooks and the task share. */
+export interface HookApi extends DocumentReader {
+	readonly taskId: TaskId;
+	readonly conversationId: ConversationId;
+	/** `HarnessOptions.models`. */
+	readonly models: Models;
+	memo<T extends JsonValue>(name: string, context: Context): Promise<T | undefined>;
+	memo<T extends JsonValue>(name: string, candidate: T, context: Context): Promise<T>;
+}
+
+export type HookResult<T> = T | undefined | Promise<T | undefined>;
+
+/** Hooks of the built-in generation task. */
+export interface GenerationHooks {
+	/** Before every request attempt, including recovery; the result is used for that request only. */
+	beforeRequest(
+		request: { readonly messages: readonly Message[] },
+		api: HookApi,
+		context: Context,
+	): HookResult<{ readonly messages: readonly Message[] }>;
+	/** Every terminal provider message, before classification. */
+	afterResponse(message: AssistantMessage, api: HookApi, context: Context): void | Promise<void>;
+	/** A final answer; the first `continue` appends a user message and continues the run. */
+	onYield(answer: AssistantMessage, api: HookApi, context: Context): HookResult<{ readonly continue: UserInput }>;
+	/** After every tool of the round is terminal; `results` are the round's result entries in call order. */
+	afterTools(assistant: EntryId, results: readonly EntryId[], api: HookApi, context: Context): void | Promise<void>;
+}
+
+/** A call as tool hooks see it; `parent` is set for a nested call, one a running tool made through `executeTool()`. */
+export type ToolHookCall = ToolCall & {
+	readonly parent?: { readonly taskId: TaskId; readonly callId: string };
+};
+
+/** Hooks of the built-in tool task, for model-issued and nested calls. */
+export interface ToolHooks {
+	/** Before intent; the first `block` wins, otherwise `arguments` replace the call's arguments. A throw blocks. */
+	beforeTool(
+		call: ToolHookCall,
+		api: HookApi,
+		context: Context,
+	): HookResult<{ readonly arguments?: JsonObject; readonly block?: string }>;
+	/** After execution, before the result is committed; replaces the result. */
+	afterTool(
+		call: ToolHookCall,
+		result: ToolExecutionResult,
+		api: HookApi,
+		context: Context,
+	): HookResult<ToolExecutionResult>;
+}
+
+/** Hooks of the built-in compaction task. */
+export interface CompactionHooks {
+	/**
+	 * After range selection, before summarizing; the first decision wins. `entries` are the active entries the summary
+	 * replaces, the head marker first, and `messages` their model context, the summarizer's source; `firstKept` is the
+	 * first entry kept verbatim.
+	 */
+	beforeCompact(
+		compaction: {
+			readonly reason: CompactionReason;
+			readonly entries: readonly EntryRecord[];
+			readonly messages: readonly Message[];
+			readonly firstKept: EntryId;
+			readonly instructions?: string;
+		},
+		api: HookApi,
+		context: Context,
+	): HookResult<{ readonly decline: true } | { readonly summary: string }>;
+}

third_party/pi/packages/durable/src/harness/usage.ts

@@ -0,0 +1,72 @@
+import { copyJson, type Draft, type JsonRepresentation } from "@earendil-works/chord";
+import type { Usage } from "@earendil-works/pi-ai";
+import { defineDoc } from "../documents.ts";
+import type { ConversationId, Tx } from "../types.ts";
+
+/** Ledger of one conversation's own spend: its entries, and compaction summarization attempts, which have none. */
+export type UsageState = {
+	/** Assistant entries and summarization attempts, keyed `provider/modelId`. */
+	models: Record<string, JsonRepresentation<Usage>>;
+	/** Tool results, keyed by tool name; their usage has no model identity. */
+	tools: Record<string, JsonRepresentation<Usage>>;
+};
+
+export const UsageDoc = defineDoc<UsageState>({
+	kind: "pi.usage",
+	version: 1,
+	scope: "conversation",
+	history: "latest",
+	fork: "initial",
+	initial: () => ({ models: {}, tools: {} }),
+	checkpointWhen: () => true,
+});
+
+/** Add `usage` to one bucket of the conversation's `pi.usage`, in the commit that records the response. */
+export async function recordUsage(
+	tx: Tx,
+	conversationId: ConversationId,
+	bucket: keyof UsageState,
+	key: string,
+	usage: Usage,
+): Promise<void> {
+	const totals = (await tx.doc(UsageDoc, conversationId))[bucket];
+	// Own keys only: a tool may be called `toString`.
+	const total = Object.hasOwn(totals, key) ? totals[key] : undefined;
+	// Providers may leave optional counters `undefined`; drafts take strict JSON.
+	if (total === undefined)
+		totals[key] = copyJson(usage, { omitUndefinedProperties: true }) as JsonRepresentation<Usage>;
+	else addUsage(total, usage);
+}
+
+/** Add every counter of `usage` to `total`; optional counters are added once either side reports them. */
+export function addUsage(total: Draft<Usage> | Usage, usage: Usage): void {
+	total.input += usage.input;
+	total.output += usage.output;
+	total.cacheRead += usage.cacheRead;
+	total.cacheWrite += usage.cacheWrite;
+	total.totalTokens += usage.totalTokens;
+	if (usage.cacheWrite1h !== undefined) total.cacheWrite1h = (total.cacheWrite1h ?? 0) + usage.cacheWrite1h;
+	if (usage.reasoning !== undefined) total.reasoning = (total.reasoning ?? 0) + usage.reasoning;
+	total.cost.input += usage.cost.input;
+	total.cost.output += usage.cost.output;
+	total.cost.cacheRead += usage.cost.cacheRead;
+	total.cost.cacheWrite += usage.cost.cacheWrite;
+	total.cost.total += usage.cost.total;
+}
+
+/** Add every bucket of `state` into `sum`. */
+export function addUsageState(sum: UsageState, state: Readonly<UsageState>): void {
+	for (const bucket of ["models", "tools"] as const) {
+		for (const [key, usage] of Object.entries(state[bucket])) {
+			const total = Object.hasOwn(sum[bucket], key) ? sum[bucket][key] : undefined;
+			// Define rather than assign: assigning a tool named `__proto__` would set the prototype.
+			if (total !== undefined) addUsage(total, usage);
+			else
+				Object.defineProperty(sum[bucket], key, {
+					value: structuredClone(usage),
+					enumerable: true,
+					writable: true,
+				});
+		}
+	}
+}

third_party/pi/packages/durable/src/harness/util.ts

@@ -0,0 +1,61 @@
+import type { Context } from "@earendil-works/chord";
+import { SessionFailed } from "../errors.ts";
+import type { Cursor, Page } from "../types.ts";
+
+/** Pending waits by key. Each settles once: through `resolve`, `rejectAll`, or cancellation of its context. */
+export class Waiters<K, T> {
+	readonly #sets = new Map<K, Set<PromiseWithResolvers<T>>>();
+
+	add(key: K, context: Context): Promise<T> {
+		const signal = context.abortSignal;
+		if (signal?.aborted) return Promise.reject(signal.reason);
+		const waiter = Promise.withResolvers<T>();
+		const set = this.#sets.get(key) ?? new Set();
+		this.#sets.set(key, set.add(waiter));
+		if (signal !== undefined) {
+			const onAbort = (): void => {
+				set.delete(waiter);
+				if (set.size === 0 && this.#sets.get(key) === set) this.#sets.delete(key);
+				waiter.reject(signal.reason);
+			};
+			signal.addEventListener("abort", onAbort, { once: true });
+			const detach = (): void => signal.removeEventListener("abort", onAbort);
+			waiter.promise.then(detach, detach);
+		}
+		return waiter.promise;
+	}
+
+	keys(): K[] {
+		return [...this.#sets.keys()];
+	}
+
+	resolve(key: K, value: T): void {
+		const set = this.#sets.get(key);
+		this.#sets.delete(key);
+		for (const waiter of set ?? []) waiter.resolve(value);
+	}
+
+	rejectAll(error: unknown): void {
+		const sets = [...this.#sets.values()];
+		this.#sets.clear();
+		for (const set of sets) for (const waiter of set) waiter.reject(error);
+	}
+}
+
+/** Every item of a paginated scan, in page order. */
+export async function scanAll<T>(scan: (cursor: Cursor | undefined) => Promise<Page<T, Cursor>>): Promise<T[]> {
+	const items: T[] = [];
+	let cursor: Cursor | undefined;
+	do {
+		const page = await scan(cursor);
+		items.push(...page.items);
+		cursor = page.next;
+	} while (cursor !== undefined);
+	return items;
+}
+
+/** What a closed Harness rejects with: `SessionFailed` with the cause when it closed because the Session failed. */
+export function closedError(session: { readonly failure: { readonly error: unknown } | undefined }): Error {
+	const failure = session.failure;
+	return failure === undefined ? new Error("Harness is closed") : new SessionFailed(failure.error);
+}

third_party/pi/packages/durable/src/harness/view.ts

@@ -0,0 +1,267 @@
+import { type AttachedReplicatedState, type Context, type JsonValue, replicatedState } from "@earendil-works/chord";
+import { withoutAbortSignal } from "@earendil-works/chord/context";
+import { applyImmutable, type NonEmptyPath, type Op, type Path } from "@earendil-works/chord/delta";
+import { CommittedStateSource, CommittedWatch } from "../session/observation.ts";
+import { contained, type SessionImpl } from "../session/session.ts";
+import type {
+	CommitPublication,
+	ConversationDocToken,
+	ConversationId,
+	ConversationRecord,
+	DocumentId,
+	EntryRecord,
+	JsonObject,
+	Storage,
+	WatchHandle,
+} from "../types.ts";
+import { AgentDoc } from "./agent.ts";
+import { activeEntries, captureContextBounds } from "./context.ts";
+import { InboxDoc } from "./inbox.ts";
+import { LiveDoc } from "./live.ts";
+import { ProviderDoc } from "./provider.ts";
+import { UsageDoc } from "./usage.ts";
+import { closedError } from "./util.ts";
+
+/** Structural mount of one conversation's active transcript and built-in documents (spec §9.3). */
+export type ConversationView = {
+	readonly conversation: ConversationRecord;
+	/** Raw active entries, as `ContextView.entries`: the head marker, then the non-head entries from its head. */
+	readonly entries: readonly EntryRecord[];
+	/** Built-in conversation documents keyed by kind; absent documents are absent. */
+	readonly docs: Readonly<Record<string, JsonObject>>;
+};
+
+/** Receives each next revision of a mount, and the Session's close. */
+export type ViewObserver = {
+	advance?(value: ConversationView, ops: readonly Op[], context: Context): void;
+	/** Every publication, after the mount took it; `ops` are the mount's, possibly none. */
+	publication?(
+		before: ConversationView,
+		after: ConversationView,
+		ops: readonly Op[],
+		publication: CommitPublication,
+		context: Context,
+	): void;
+	closeSession(failure: { readonly error: unknown } | undefined): void;
+};
+
+const MOUNTED = [
+	AgentDoc,
+	LiveDoc,
+	InboxDoc,
+	ProviderDoc,
+	UsageDoc,
+] as unknown as readonly ConversationDocToken<JsonObject>[];
+const MOUNTED_KINDS: ReadonlySet<string> = new Set(MOUNTED.map((token) => token.definition.kind));
+
+/** One conversation's mount: its current revision, the document incarnations it shows, and its observers. */
+type Mount = {
+	value: ConversationView;
+	/** Mounted incarnation and definition version per kind; another incarnation or version is set whole. */
+	readonly docs: Map<string, { id: DocumentId; version: number }>;
+	readonly observers: Set<ViewObserver>;
+};
+
+const VIEWS = new WeakMap<object, ConversationViews>();
+
+/** The view mounts of a Harness, for adapters built on them. */
+export function conversationViews(harness: object): ConversationViews {
+	const views = VIEWS.get(harness);
+	if (views === undefined) throw new Error("Not a Harness");
+	return views;
+}
+
+/**
+ * The Harness's conversation view mounts: at most one per conversation, built on the Session line by its first
+ * observer and dropped with its last. Each mount advances from the Session's commit publications, which are durable.
+ */
+export class ConversationViews {
+	readonly #session: SessionImpl;
+	readonly #storage: Storage;
+	readonly #mounts = new Map<ConversationId, Mount>();
+	#closed = false;
+
+	constructor(session: SessionImpl, storage: Storage) {
+		this.#session = session;
+		this.#storage = storage;
+		VIEWS.set(session, this);
+		session.observeCommits((publication, context) => {
+			for (const [id, mount] of this.#mounts)
+				advance(id, mount, publication, context, (error) => this.report(error));
+		});
+		session.subscribeClose(() => {
+			this.#closed = true;
+			for (const mount of this.#mounts.values())
+				for (const observer of [...mount.observers]) {
+					contained(
+						() => observer.closeSession(this.#session.failure),
+						(error) => this.report(error),
+					);
+				}
+			this.#mounts.clear();
+		});
+	}
+
+	/** Report an observer's listener error through the Harness. */
+	report(error: unknown): void {
+		this.#session.report(error);
+	}
+
+	/** A disposable read-only Chord state of the view. */
+	async state(id: ConversationId, context: Context): Promise<AttachedReplicatedState<ConversationView>> {
+		const { observer, detach } = await this.attach(
+			id,
+			(value, release) => new CommittedStateSource<ConversationView>(value, release),
+			context,
+		);
+		try {
+			return replicatedState(observer, { onError: (error) => this.#session.report(error) });
+		} catch (error) {
+			detach();
+			throw error;
+		}
+	}
+
+	/** A serialized exact-frame watch of the view; cancelling `context` stops it. */
+	async watch(id: ConversationId, context: Context): Promise<WatchHandle<ConversationView>> {
+		const { observer } = await this.attach(
+			id,
+			(value, release) =>
+				new CommittedWatch<ConversationView>(value, release, (error) => this.#session.report(error)),
+			context,
+		);
+		const signal = context.abortSignal;
+		if (signal?.aborted) {
+			observer.cancel();
+			throw signal.reason;
+		}
+		if (signal !== undefined) observer.observeCancellation(signal);
+		return observer;
+	}
+
+	/**
+	 * Register an observer created from the current revision, atomically on the Session line: it sees every later
+	 * publication and nothing earlier. `create` may read committed Storage, still on the line. `release` drops it, and
+	 * the mount with its last observer.
+	 */
+	attach<O extends ViewObserver>(
+		id: ConversationId,
+		create: (value: ConversationView, release: () => void, storage: Storage) => O | Promise<O>,
+		context: Context,
+	): Promise<{ observer: O; detach: () => void }> {
+		return this.#session.readOnLine(async () => {
+			const mount = this.#mounts.get(id) ?? (await this.#build(id, context));
+			const detach = (): void => {
+				mount.observers.delete(observer);
+				if (mount.observers.size === 0 && this.#mounts.get(id) === mount) this.#mounts.delete(id);
+			};
+			const observer = await create(mount.value, detach, this.#storage);
+			// Close or cancellation may begin while the mount hydrates; register nothing then.
+			if (this.#closed) throw closedError(this.#session);
+			context.abortSignal?.throwIfAborted();
+			this.#mounts.set(id, mount);
+			mount.observers.add(observer);
+			return { observer, detach };
+		});
+	}
+
+	async #build(id: ConversationId, context: Context): Promise<Mount> {
+		const conversation = await this.#storage.conversation(id, context);
+		if (conversation === undefined) throw new Error(`Conversation ${id} does not exist`);
+		const bounds = await captureContextBounds(this.#storage, id, context);
+		const entries = await activeEntries(this.#storage, id, bounds, context);
+		const docs: Record<string, JsonObject> = {};
+		const incarnations = new Map<string, { id: DocumentId; version: number }>();
+		for (const token of MOUNTED) {
+			const loaded = await this.#session.conversationDocumentOnLine(token, id, context);
+			if (loaded === undefined) continue;
+			docs[token.definition.kind] = loaded.value;
+			incarnations.set(token.definition.kind, { id: loaded.record.id, version: loaded.version });
+		}
+		return { value: { conversation, entries, docs }, docs: incarnations, observers: new Set() };
+	}
+}
+
+/**
+ * Derive the mount's operations from one publication, apply them, and hand the revision to every observer; one that
+ * throws is reported through `report`, and the others still get it.
+ */
+function advance(
+	id: ConversationId,
+	mount: Mount,
+	publication: CommitPublication,
+	context: Context,
+	report: (error: unknown) => void,
+): void {
+	const docOps: Op[] = [];
+	const entryOps: Op[] = [];
+	let entries = mount.value.entries;
+	// Entry writes are published in ID order.
+	for (const change of publication.changes) {
+		if (change.type === "entry" && change.value.conversationId === id) {
+			const entry = change.value;
+			const value = entry as unknown as JsonValue;
+			if (entry.head === undefined) {
+				entryOps.push(["p", ["entries"], entries.length, 0, [value]]);
+				entries = [...entries, entry];
+				continue;
+			}
+			// A head marker keeps the non-head entries from its head, which are always a suffix, and goes in front.
+			const target = entry.head;
+			let kept = entries.findIndex((candidate) => candidate.head === undefined && candidate.id >= target);
+			if (kept < 0) kept = entries.length;
+			entryOps.push(["p", ["entries"], 0, kept, [value]]);
+			entries = [entry, ...entries.slice(kept)];
+			continue;
+		}
+		if (change.type !== "document" || change.conversationId !== id) continue;
+		const kind = change.record.kind;
+		if (!MOUNTED_KINDS.has(kind) || change.record.key !== undefined) continue;
+		const path: NonEmptyPath = ["docs", kind];
+		const mounted = mount.docs.get(kind);
+		if (change.value === null) {
+			if (mounted?.id !== change.record.id) continue;
+			mount.docs.delete(kind);
+			docOps.push(["d", path]);
+		} else if (mounted?.id === change.record.id && mounted.version === change.version) {
+			for (const op of change.ops) docOps.push(prefixed(op, path));
+		} else {
+			mount.docs.set(kind, { id: change.record.id, version: change.version! });
+			docOps.push(["s", path, change.value]);
+		}
+	}
+	const before = mount.value;
+	const ops = [...docOps, ...entryOps];
+	const frameContext = withoutAbortSignal(context);
+	if (ops.length > 0) {
+		const value = docOps.length === 0 ? mount.value : applyImmutable(mount.value, docOps);
+		mount.value = entries === value.entries ? value : { ...value, entries };
+		for (const observer of [...mount.observers]) {
+			contained(() => observer.advance?.(mount.value, ops, frameContext), report);
+		}
+	}
+	for (const observer of [...mount.observers]) {
+		contained(() => observer.publication?.(before, mount.value, ops, publication, frameContext), report);
+	}
+}
+
+/** `op` moved under `prefix`; a root replacement becomes a set of the prefix. */
+function prefixed(op: Op, prefix: NonEmptyPath): Op {
+	const at = (path: Path): NonEmptyPath => [...prefix, ...path] as unknown as NonEmptyPath;
+	switch (op[0]) {
+		case "r":
+			return ["s", prefix, op[1]];
+		case "p":
+			return ["p", at(op[1]), op[2], op[3], op[4]];
+		case "m":
+			return ["m", at(op[1]), op[2]];
+		case "s":
+			return ["s", at(op[1]), op[2]];
+		case "d":
+			return ["d", at(op[1])];
+		case "a":
+			return ["a", at(op[1]), op[2]];
+		case "t":
+			return ["t", at(op[1]), op[2]];
+	}
+}

third_party/pi/packages/durable/src/ids.ts

@@ -0,0 +1,11 @@
+import type { Id, Seq } from "./types.ts";
+
+/** Apply an erased ID brand at a trusted numeric allocation or decoding boundary. */
+export function idFromNumber<I extends Id<string>>(value: number): I {
+	return value as I;
+}
+
+/** Apply the erased commit-sequence brand at a trusted storage boundary. */
+export function seqFromNumber(value: number): Seq {
+	return value as Seq;
+}

third_party/pi/packages/durable/src/images/cloudflare.ts

@@ -0,0 +1,10 @@
+// Wrangler's default `CompiledWasm` rule turns this import into a `WebAssembly.Module`; Workers cannot compile
+// WebAssembly from bytes at runtime. Other bundlers do not support it: use `createPhotonImages()` there.
+import photonWasm from "@silvia-odwyer/photon/photon_rs_bg.wasm";
+import type { ImageProcessor } from "../tools/image-processor.ts";
+import { createPhotonImages } from "./index.ts";
+
+/** `createPhotonImages()` for Cloudflare Workers and Durable Objects, built with Wrangler. */
+export function createCloudflarePhotonImages(): ImageProcessor {
+	return createPhotonImages(photonWasm);
+}

third_party/pi/packages/durable/src/images/exif.ts

@@ -0,0 +1,70 @@
+// EXIF orientation of JPEG and WebP files, read from the bytes without decoding the image.
+
+/** The EXIF orientation of a JPEG or WebP, 1 to 8; 1 when absent or unreadable. */
+export function exifOrientation(bytes: Uint8Array): number {
+	const tiff = tiffStart(bytes);
+	if (tiff < 0 || tiff + 8 > bytes.length) return 1;
+	const little = bytes[tiff] === 0x49 && bytes[tiff + 1] === 0x49;
+	const u16 = (at: number): number =>
+		little ? bytes[at]! | (bytes[at + 1]! << 8) : (bytes[at]! << 8) | bytes[at + 1]!;
+	const u32 = (at: number): number =>
+		little
+			? (bytes[at]! | (bytes[at + 1]! << 8) | (bytes[at + 2]! << 16) | (bytes[at + 3]! << 24)) >>> 0
+			: ((bytes[at]! << 24) | (bytes[at + 1]! << 16) | (bytes[at + 2]! << 8) | bytes[at + 3]!) >>> 0;
+	const directory = tiff + u32(tiff + 4);
+	if (directory + 2 > bytes.length) return 1;
+	const entries = u16(directory);
+	for (let index = 0; index < entries; index++) {
+		const entry = directory + 2 + index * 12;
+		if (entry + 12 > bytes.length) return 1;
+		// Tag 0x0112 is Orientation, a SHORT whose value sits in the entry.
+		if (u16(entry) === 0x0112) {
+			const value = u16(entry + 8);
+			return value >= 1 && value <= 8 ? value : 1;
+		}
+	}
+	return 1;
+}
+
+/** Where the TIFF header of a JPEG's APP1 or a WebP's EXIF chunk starts; -1 when there is none. */
+function tiffStart(bytes: Uint8Array): number {
+	if (bytes[0] === 0xff && bytes[1] === 0xd8) {
+		for (let offset = 2; offset + 4 <= bytes.length; ) {
+			if (bytes[offset] !== 0xff) return -1;
+			const marker = bytes[offset + 1]!;
+			if (marker === 0xff) {
+				offset++;
+				continue;
+			}
+			if (marker === 0xe1 && isExifHeader(bytes, offset + 4)) return offset + 10;
+			offset += 2 + ((bytes[offset + 2]! << 8) | bytes[offset + 3]!);
+		}
+		return -1;
+	}
+	if (ascii(bytes, 0, "RIFF") && ascii(bytes, 8, "WEBP")) {
+		for (let offset = 12; offset + 8 <= bytes.length; ) {
+			const size =
+				(bytes[offset + 4]! |
+					(bytes[offset + 5]! << 8) |
+					(bytes[offset + 6]! << 16) |
+					(bytes[offset + 7]! << 24)) >>>
+				0;
+			// Some WebP files prefix the TIFF header with "Exif\0\0".
+			if (ascii(bytes, offset, "EXIF")) return isExifHeader(bytes, offset + 8) ? offset + 14 : offset + 8;
+			// RIFF chunks are padded to an even size.
+			offset += 8 + size + (size % 2);
+		}
+	}
+	return -1;
+}
+
+function isExifHeader(bytes: Uint8Array, offset: number): boolean {
+	return ascii(bytes, offset, "Exif") && bytes[offset + 4] === 0 && bytes[offset + 5] === 0;
+}
+
+function ascii(bytes: Uint8Array, offset: number, text: string): boolean {
+	for (let index = 0; index < text.length; index++) {
+		if (bytes[offset + index] !== text.charCodeAt(index)) return false;
+	}
+	return true;
+}

third_party/pi/packages/durable/src/images/index.ts

@@ -0,0 +1,140 @@
+// The package declares only `module`, which bundlers use but Node does not: import the file itself.
+import {
+	fliph,
+	flipv,
+	initSync,
+	PhotonImage,
+	resize,
+	SamplingFilter,
+	type SyncInitInput,
+} from "@silvia-odwyer/photon/photon_rs.js";
+import type { ImageLimits, ImageProcessor, PreparedImage } from "../tools/image-processor.ts";
+import { base64Length, INLINE_IMAGE_TYPES, toBase64 } from "../tools/image-processor.ts";
+import { exifOrientation } from "./exif.ts";
+
+/** JPEG qualities tried in order. */
+const JPEG_QUALITIES = [80, 70, 55, 40];
+
+/**
+ * An `ImageProcessor` on Photon (Rust compiled to WebAssembly), for `createCodingTools({ images })`. `wasm` is
+ * Photon's `photon_rs_bg.wasm` as bytes or as a compiled module: `@earendil-works/pi-durable/images/node` reads it from
+ * the installed package, `@earendil-works/pi-durable/images/cloudflare` imports it as a Workers module. Elsewhere, such
+ * as a browser, fetch the file and pass its bytes. Photon loads once per process; later calls reuse the first `wasm`.
+ *
+ * An image already upright, in a format providers take inline, and within the limits is returned as it is, once it
+ * decodes. Otherwise it is turned upright by its EXIF orientation, scaled to the
+ * maximum width and height, and encoded as PNG or JPEG at falling qualities (JPEG first for JPEG sources), then at
+ * smaller dimensions, 3/4 at a time, until the base64 fits. Formats providers do not take inline (BMP) become PNG.
+ *
+ * The work runs on the calling thread: about a second and a half for a 12-megapixel photo. Decoding takes the image's
+ * RGBA size in WebAssembly memory, which does not shrink again; leave the processor out on memory-constrained hosts.
+ */
+export function createPhotonImages(wasm: SyncInitInput): ImageProcessor {
+	initSync({ module: wasm });
+	return { prepare: async (bytes, mimeType, limits) => prepare(bytes, mimeType, limits) };
+}
+
+function prepare(bytes: Uint8Array, mimeType: string, limits: ImageLimits): PreparedImage | undefined {
+	const orientation = exifOrientation(bytes);
+	const inline = INLINE_IMAGE_TYPES.has(mimeType);
+	let image: PhotonImage | undefined;
+	try {
+		// Decoded even when it is returned as it is: a damaged file is an error, not an image the provider rejects.
+		image = PhotonImage.new_from_byteslice(bytes);
+		if (
+			inline &&
+			orientation === 1 &&
+			image.get_width() <= limits.maxWidth &&
+			image.get_height() <= limits.maxHeight &&
+			base64Length(bytes.byteLength) <= limits.maxBytes
+		) {
+			return { data: toBase64(bytes), mimeType };
+		}
+		if (orientation >= 5) {
+			const upright = transposed(image, orientation);
+			// Freed first, so the upright copy reuses its WebAssembly memory.
+			image.free();
+			image = undefined;
+			image = new PhotonImage(upright.pixels, upright.width, upright.height);
+		} else if (orientation !== 1) {
+			if (orientation !== 4) fliph(image);
+			if (orientation !== 2) flipv(image);
+		}
+		const width = image.get_width();
+		const height = image.get_height();
+		const scale = Math.min(1, limits.maxWidth / width, limits.maxHeight / height);
+		let target = { width: Math.max(1, Math.round(width * scale)), height: Math.max(1, Math.round(height * scale)) };
+		for (;;) {
+			const encoded = encodeWithin(image, target, limits.maxBytes, mimeType === "image/jpeg");
+			if (encoded !== undefined) {
+				const resized =
+					target.width === width && target.height === height
+						? {}
+						: { resized: { from: { width, height }, to: target } };
+				const converted = encoded.mimeType === mimeType ? {} : { convertedFrom: mimeType };
+				return { ...encoded, ...converted, ...resized };
+			}
+			if (target.width === 1 && target.height === 1) return undefined;
+			target = {
+				width: Math.max(1, Math.floor(target.width * 0.75)),
+				height: Math.max(1, Math.floor(target.height * 0.75)),
+			};
+		}
+	} catch {
+		// Photon throws for bytes it cannot decode.
+		return undefined;
+	} finally {
+		image?.free();
+	}
+}
+
+/**
+ * `image` at `size` in the first encoding whose base64 fits `maxBytes`: PNG, then JPEG at falling qualities, or the
+ * JPEG qualities first when `jpegFirst`; `undefined` if none fits.
+ */
+function encodeWithin(
+	image: PhotonImage,
+	size: { readonly width: number; readonly height: number },
+	maxBytes: number,
+	jpegFirst: boolean,
+): { readonly data: string; readonly mimeType: string } | undefined {
+	const same = size.width === image.get_width() && size.height === image.get_height();
+	const scaled = same ? image : resize(image, size.width, size.height, SamplingFilter.Lanczos3);
+	try {
+		const png = { mimeType: "image/png", encode: () => scaled.get_bytes() };
+		const jpegs = JPEG_QUALITIES.map((quality) => ({
+			mimeType: "image/jpeg",
+			encode: () => scaled.get_bytes_jpeg(quality),
+		}));
+		for (const candidate of jpegFirst ? [...jpegs, png] : [png, ...jpegs]) {
+			const bytes = candidate.encode();
+			if (base64Length(bytes.byteLength) <= maxBytes) return { data: toBase64(bytes), mimeType: candidate.mimeType };
+		}
+		return undefined;
+	} finally {
+		if (!same) scaled.free();
+	}
+}
+
+/**
+ * The RGBA pixels of `image` turned upright for EXIF `orientation` 5 to 8, the orientations that swap width and
+ * height. Photon's own `rotate` drops pixels at right angles, so the pixels are moved here.
+ */
+function transposed(
+	image: PhotonImage,
+	orientation: number,
+): { readonly pixels: Uint8Array; readonly width: number; readonly height: number } {
+	const width = image.get_width();
+	const height = image.get_height();
+	const source = image.get_raw_pixels();
+	const pixels = new Uint8Array(source.length);
+	for (let y = 0; y < height; y++) {
+		for (let x = 0; x < width; x++) {
+			// Where (x, y) lands in the upright image, which is `height` wide.
+			const outX = orientation === 5 || orientation === 8 ? y : height - 1 - y;
+			const outY = orientation === 5 || orientation === 6 ? x : width - 1 - x;
+			pixels.set(source.subarray((y * width + x) * 4, (y * width + x) * 4 + 4), (outY * height + outX) * 4);
+		}
+	}
+	return { pixels, width: height, height: width };
+}

third_party/pi/packages/durable/src/images/node-worker.ts

@@ -0,0 +1,35 @@
+// The worker thread of `createNodePhotonImages()`: Photon prepares images here, off the caller's event loop.
+import { readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { parentPort } from "node:worker_threads";
+import type { ImageLimits, PreparedImage } from "../tools/image-processor.ts";
+import { createPhotonImages } from "./index.ts";
+
+/** One request: the image to prepare, and its ID for the reply. */
+export type PrepareRequest = {
+	readonly id: number;
+	readonly bytes: Uint8Array;
+	readonly mimeType: string;
+	readonly limits: ImageLimits;
+};
+
+/** The reply to a request: its result, or why it failed. */
+export type PrepareReply =
+	| { readonly id: number; readonly result: PreparedImage | undefined }
+	| { readonly id: number; readonly error: string };
+
+const port = parentPort;
+if (port === null) throw new Error("The Photon image worker needs a parent thread");
+const images = createPhotonImages(
+	readFileSync(createRequire(import.meta.url).resolve("@silvia-odwyer/photon/photon_rs_bg.wasm")),
+);
+port.on("message", (request: PrepareRequest) => {
+	void images.prepare(request.bytes, request.mimeType, request.limits).then(
+		(result) => port.postMessage({ id: request.id, result } satisfies PrepareReply),
+		(error: unknown) =>
+			port.postMessage({
+				id: request.id,
+				error: error instanceof Error ? error.message : String(error),
+			} satisfies PrepareReply),
+	);
+});

third_party/pi/packages/durable/src/images/node.ts

@@ -0,0 +1,130 @@
+import { readFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { Worker } from "node:worker_threads";
+import type { ImageLimits, ImageProcessor, PreparedImage } from "../tools/image-processor.ts";
+import { createPhotonImages } from "./index.ts";
+import type { PrepareReply, PrepareRequest } from "./node-worker.ts";
+
+export type NodePhotonImagesOptions = {
+	/**
+	 * Prepare images in a worker thread, so a large photo does not block the event loop for the second or two Photon
+	 * takes. Default `true`; `false` prepares them on the calling thread.
+	 */
+	readonly worker?: boolean;
+	/**
+	 * How long the worker stays after its last image before it exits, freeing Photon's WebAssembly memory, which never
+	 * shrinks while it lives. The next image starts a new one. Default 30 s.
+	 */
+	readonly idleMs?: number;
+};
+
+/**
+ * `createPhotonImages()` for Node and Bun, with Photon's WebAssembly read from the installed `@silvia-odwyer/photon`,
+ * by default in a worker thread that keeps the process alive only while it prepares an image. A worker that cannot
+ * start falls back to the calling thread. A single-file binary cannot resolve the package: read its own copy of
+ * `photon_rs_bg.wasm` and call `createPhotonImages()`.
+ */
+export async function createNodePhotonImages(options: NodePhotonImagesOptions = {}): Promise<ImageProcessor> {
+	if (options.worker === false) return inProcess();
+	const idleMs = options.idleMs ?? 30_000;
+	let thread: PhotonThread | undefined;
+	let fallback: Promise<ImageProcessor> | undefined;
+	return {
+		async prepare(bytes, mimeType, limits) {
+			if (fallback !== undefined) return (await fallback).prepare(bytes, mimeType, limits);
+			try {
+				thread ??= new PhotonThread(idleMs, () => {
+					thread = undefined;
+				});
+				return await thread.prepare(bytes, mimeType, limits);
+			} catch (error) {
+				if (!(error instanceof WorkerUnavailable)) throw error;
+				// No worker on this platform or build: prepare here from now on.
+				fallback = inProcess();
+				return (await fallback).prepare(bytes, mimeType, limits);
+			}
+		},
+	};
+}
+
+async function inProcess(): Promise<ImageProcessor> {
+	const path = createRequire(import.meta.url).resolve("@silvia-odwyer/photon/photon_rs_bg.wasm");
+	return createPhotonImages(await readFile(path));
+}
+
+/** The worker could not start, as opposed to failing on one image. */
+class WorkerUnavailable extends Error {}
+
+/** One worker thread and the requests it has not answered; it exits after `idleMs` without requests. */
+class PhotonThread {
+	readonly #worker: Worker;
+	readonly #pending = new Map<number, PromiseWithResolvers<PreparedImage | undefined>>();
+	readonly #idleMs: number;
+	readonly #onExit: () => void;
+	#nextId = 0;
+	#idle: ReturnType<typeof setTimeout> | undefined;
+	/** Whether the worker answered once: a failure before that means it is unavailable. */
+	#started = false;
+
+	constructor(idleMs: number, onExit: () => void) {
+		this.#idleMs = idleMs;
+		this.#onExit = onExit;
+		// Run from source under a TypeScript runtime, as in tests; built, the worker sits next to this file as JS.
+		const file = import.meta.url.endsWith(".ts") ? "./node-worker.ts" : "./node-worker.js";
+		try {
+			this.#worker = new Worker(new URL(file, import.meta.url));
+		} catch (error) {
+			throw new WorkerUnavailable("The Photon image worker cannot start", { cause: error });
+		}
+		this.#worker.unref();
+		this.#worker.on("message", (reply: PrepareReply) => {
+			this.#started = true;
+			const request = this.#pending.get(reply.id);
+			if (request === undefined) return;
+			this.#pending.delete(reply.id);
+			if ("error" in reply) request.reject(new Error(reply.error));
+			else request.resolve(reply.result);
+			this.#settled();
+		});
+		const fail = (error: unknown): void => {
+			const failure = this.#started
+				? error
+				: new WorkerUnavailable("The Photon image worker cannot start", { cause: error });
+			for (const request of this.#pending.values()) request.reject(failure);
+			this.#pending.clear();
+			this.#exit();
+		};
+		this.#worker.on("error", fail);
+		this.#worker.on("exit", (code) => fail(new Error(`The Photon image worker exited with code ${code}`)));
+	}
+
+	prepare(bytes: Uint8Array, mimeType: string, limits: ImageLimits): Promise<PreparedImage | undefined> {
+		const id = this.#nextId++;
+		const request = Promise.withResolvers<PreparedImage | undefined>();
+		this.#pending.set(id, request);
+		if (this.#idle !== undefined) clearTimeout(this.#idle);
+		this.#idle = undefined;
+		// A pending image keeps the process alive, as one prepared on the calling thread would.
+		this.#worker.ref();
+		// A copy, transferred: the caller keeps its bytes.
+		const copy = bytes.slice();
+		this.#worker.postMessage({ id, bytes: copy, mimeType, limits } satisfies PrepareRequest, [copy.buffer]);
+		return request.promise;
+	}
+
+	#settled(): void {
+		if (this.#pending.size > 0) return;
+		this.#worker.unref();
+		this.#idle = setTimeout(() => {
+			this.#exit();
+			void this.#worker.terminate();
+		}, this.#idleMs);
+		this.#idle.unref();
+	}
+
+	#exit(): void {
+		if (this.#idle !== undefined) clearTimeout(this.#idle);
+		this.#idle = undefined;
+		this.#onExit();
+	}
+}

third_party/pi/packages/durable/src/images/photon-wasm.d.ts

@@ -0,0 +1,5 @@
+// Imported by cloudflare.ts. Wrangler's default `CompiledWasm` rule makes a .wasm import a compiled module.
+declare module "@silvia-odwyer/photon/photon_rs_bg.wasm" {
+	const module: WebAssembly.Module;
+	export default module;
+}

third_party/pi/packages/durable/src/index.ts

@@ -0,0 +1,205 @@
+export { defineDoc, defineDocFamily } from "./documents.ts";
+export {
+	AssistantEntry,
+	CompactionEntry,
+	defineEntry,
+	ResetEntry,
+	SystemEntry,
+	ToolResultEntry,
+	UserEntry,
+} from "./entries.ts";
+export { ConversationBusy, ReadAfterWrite, SessionFailed, StorageRequestError } from "./errors.ts";
+export {
+	AgentDoc,
+	configure,
+	DEFAULT_COMPACTION_POLICY,
+	DEFAULT_PROGRESS_POLICY,
+	DEFAULT_RETRY_POLICY,
+} from "./harness/agent.ts";
+export {
+	type CompactionCheckpoint,
+	type CompactionInput,
+	CompactionTask,
+	type SummaryRequest,
+} from "./harness/compaction.ts";
+export { defineExtension, defineTool, hook, section, wrapSection, wrapTool } from "./harness/define.ts";
+export {
+	type AgentEvent,
+	type AgentEventStream,
+	type MessageChange,
+	type SnapshotEvent,
+	type ToolEventCall,
+	watchEvents,
+} from "./harness/events.ts";
+export {
+	type GenerationCheckpoint,
+	type GenerationInput,
+	type GenerationResult,
+	GenerationTask,
+} from "./harness/generation.ts";
+export { Harness } from "./harness/harness.ts";
+export { InboxDoc, type InboxItem, type InboxState } from "./harness/inbox.ts";
+export {
+	type CompactionFailure,
+	type CompactionStatus,
+	LiveDoc,
+	type LiveState,
+	type NestedToolSlot,
+	type NestedToolSummary,
+	type ToolSlot,
+} from "./harness/live.ts";
+export { ProviderDoc, type ProviderState } from "./harness/provider.ts";
+export { createRegistry } from "./harness/registry.ts";
+export type { TaskGraph, TaskGraphNode, TaskGraphState, TaskGraphWatch } from "./harness/task-graph.ts";
+export {
+	NestedCallDoc,
+	type NestedCallState,
+	ToolTask,
+	type ToolTaskCheckpoint,
+	type ToolTaskInput,
+	type ToolTaskResult,
+} from "./harness/tool.ts";
+export type {
+	Agent,
+	AgentChange,
+	AgentState,
+	AnyTask,
+	CompactionHooks,
+	CompactionPolicy,
+	CompactionReason,
+	CompactionResult,
+	ContextView,
+	Conversation,
+	ConversationAbortOptions,
+	ConversationCreateOptions,
+	ConversationHandle,
+	ConversationInit,
+	ConversationRetryPolicy,
+	ConversationStreamOptions,
+	ConversationWatch,
+	EnvTarget,
+	Extension,
+	GenerationHooks,
+	HarnessInspection,
+	HarnessOptions,
+	HarnessSettings,
+	HookApi,
+	HookRegistration,
+	HookResult,
+	HooksOf,
+	InputEntry,
+	InputSubmissionDraft,
+	ModelRef,
+	NestedToolExecutionResult,
+	ProgressPolicy,
+	PromptInput,
+	PromptSection,
+	QueueMode,
+	Registry,
+	RegistryReader,
+	RegistrySnapshot,
+	Settings,
+	SettledSubmissionRecord,
+	SettledTask,
+	Submission,
+	SubmissionDraft,
+	TaskInspection,
+	ToolControl,
+	ToolDiagnostic,
+	ToolExecutionApi,
+	ToolExecutionMode,
+	ToolExecutionResult,
+	ToolHookCall,
+	ToolHooks,
+	ToolRegistration,
+	UserInput,
+	Wrap,
+} from "./harness/types.ts";
+export { UsageDoc, type UsageState } from "./harness/usage.ts";
+export type { ConversationView } from "./harness/view.ts";
+export { createSession } from "./session/session.ts";
+export { MemoryStorage } from "./storage/memory.ts";
+export { defineTask } from "./tasks.ts";
+export type {
+	CheckpointInfo,
+	CommitChange,
+	CommitPublication,
+	CommonDocDefinition,
+	ContextEdit,
+	ConversationDocFamilyToken,
+	ConversationDocToken,
+	ConversationId,
+	ConversationOwnership,
+	ConversationQuery,
+	ConversationRecord,
+	Cursor,
+	DocDefinition,
+	DocFamilyDefinition,
+	DocFamilyToken,
+	DocToken,
+	DocumentAddress,
+	DocumentCommitChange,
+	DocumentContent,
+	DocumentCopySource,
+	DocumentCreate,
+	DocumentId,
+	DocumentObserver,
+	DocumentPoint,
+	DocumentQuery,
+	DocumentReader,
+	DocumentRecord,
+	DocumentSemantics,
+	DocumentState,
+	DocumentWatch,
+	Entry,
+	EntryDraft,
+	EntryId,
+	EntryQuery,
+	EntryRecord,
+	HookRunner,
+	Id,
+	JoinPolicy,
+	JsonObject,
+	LatestConversationSemantics,
+	NextTaskState,
+	Page,
+	PhaseHandler,
+	RewindableConversationDocFamilyToken,
+	RewindableConversationDocToken,
+	RewindableConversationSemantics,
+	RunningTask,
+	ScanOrder,
+	Seq,
+	Session,
+	SessionDocFamilyToken,
+	SessionDocToken,
+	SessionEnd,
+	Storage,
+	StorageWrite,
+	StoredDocument,
+	SubmissionCreate,
+	SubmissionId,
+	SubmissionQuery,
+	SubmissionRecord,
+	SubmissionSettlement,
+	TableCommitChange,
+	Task,
+	TaskDefinition,
+	TaskDocFamilyToken,
+	TaskDocToken,
+	TaskId,
+	TaskOptions,
+	TaskOutcome,
+	TaskOutcomeError,
+	TaskOwnership,
+	TaskQuery,
+	TaskRecord,
+	TaskRuntime,
+	TaskState,
+	Tx,
+	TypedEntry,
+	TypedEntryDraft,
+	WatchEnd,
+	WatchHandle,
+} from "./types.ts";
+export { ROOT_CONVERSATION_ID } from "./types.ts";

third_party/pi/packages/durable/src/search.ts

@@ -0,0 +1,85 @@
+/**
+ * BM25 tool search for code mode's `searchTools()` and the `tool_search` tool, a copy of the coding agent's tool search
+ * ranking: durable keeps its own copy of what it needs from the coding agent.
+ */
+
+export type SearchDocument = { readonly name: string; readonly text: string };
+
+const STOP_WORDS: ReadonlySet<string> = new Set([
+	"a",
+	"an",
+	"and",
+	"are",
+	"as",
+	"at",
+	"be",
+	"by",
+	"for",
+	"from",
+	"in",
+	"is",
+	"it",
+	"of",
+	"on",
+	"or",
+	"that",
+	"the",
+	"this",
+	"to",
+	"with",
+]);
+
+/** Naive singular form, so `issues` matches `issue` and `searches` matches `search`. */
+function stem(term: string): string {
+	if (term.length > 4 && term.endsWith("ies")) return `${term.slice(0, -3)}y`;
+	if (term.length > 4 && /(ches|shes|sses|xes|zes)$/.test(term)) return term.slice(0, -2);
+	if (term.length > 3 && term.endsWith("s") && !term.endsWith("ss")) return term.slice(0, -1);
+	return term;
+}
+
+/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */
+export function tokenize(text: string): string[] {
+	return text
+		.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+		.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
+		.toLowerCase()
+		.split(/[^a-z0-9]+/)
+		.filter((term) => term.length > 0 && !STOP_WORDS.has(term))
+		.map(stem);
+}
+
+/** Okapi BM25 (k1 1.2, b 0.75): the names of the best matches, best first; ties keep document order. */
+export function rankDocuments(query: string, documents: readonly SearchDocument[], limit: number): string[] {
+	const k1 = 1.2;
+	const b = 0.75;
+	const queryTerms = [...new Set(tokenize(query))];
+	if (queryTerms.length === 0 || documents.length === 0 || limit <= 0) return [];
+	const termCounts = documents.map((document) => {
+		const counts = new Map<string, number>();
+		for (const term of tokenize(document.text)) counts.set(term, (counts.get(term) ?? 0) + 1);
+		return counts;
+	});
+	const lengths = termCounts.map((counts) => [...counts.values()].reduce((sum, count) => sum + count, 0));
+	const averageLength = lengths.reduce((sum, length) => sum + length, 0) / documents.length || 1;
+	const idf = new Map(
+		queryTerms.map((term) => {
+			const frequency = termCounts.filter((counts) => counts.has(term)).length;
+			return [term, Math.log(1 + (documents.length - frequency + 0.5) / (frequency + 0.5))] as const;
+		}),
+	);
+	const matches: { name: string; score: number }[] = [];
+	documents.forEach((document, index) => {
+		let score = 0;
+		for (const term of queryTerms) {
+			const count = termCounts[index]!.get(term);
+			if (!count) continue;
+			const norm = k1 * (1 - b + (b * lengths[index]!) / averageLength);
+			score += (idf.get(term) ?? 0) * ((count * (k1 + 1)) / (count + norm));
+		}
+		if (score > 0) matches.push({ name: document.name, score });
+	});
+	return matches
+		.sort((left, right) => right.score - left.score)
+		.slice(0, limit)
+		.map((match) => match.name);
+}

third_party/pi/packages/durable/src/session/forks.ts

@@ -0,0 +1,97 @@
+import type { Context } from "@earendil-works/chord";
+import { addressId } from "../documents.ts";
+import type {
+	ConversationId,
+	Cursor,
+	DocumentCopySource,
+	DocumentCreate,
+	DocumentId,
+	DocumentPoint,
+	DocumentRecord,
+	EntryId,
+	Storage,
+} from "../types.ts";
+
+const SCAN_PAGE_SIZE = 256;
+
+type ForkPolicy = "asOf" | "current";
+
+/** One definition-free document copy to create with a forked conversation. */
+export type ForkDocumentCopy = {
+	readonly record: DocumentCreate;
+	readonly source: DocumentCopySource;
+};
+
+/** Select every persisted conversation document copied by one fork. */
+export async function prepareForkDocumentCopies(
+	storage: Storage,
+	parentConversationId: ConversationId,
+	at: EntryId,
+	childConversationId: ConversationId,
+	context: Context,
+): Promise<readonly ForkDocumentCopy[]> {
+	const entry = await storage.entry(parentConversationId, at, context);
+	if (entry === undefined) throw new Error(`Entry ${at} is not visible from conversation ${parentConversationId}`);
+
+	const copies: ForkDocumentCopy[] = [];
+	const copiedAddresses = new Set<string>();
+	await collectCopies(
+		storage,
+		{ kind: "conversation", conversationId: entry.entry.conversationId },
+		entry.commitSeq,
+		"asOf",
+		childConversationId,
+		copies,
+		copiedAddresses,
+		context,
+	);
+	await collectCopies(
+		storage,
+		{ kind: "conversation", conversationId: parentConversationId },
+		"current",
+		"current",
+		childConversationId,
+		copies,
+		copiedAddresses,
+		context,
+	);
+	return copies;
+}
+
+async function collectCopies(
+	storage: Storage,
+	scope: Extract<DocumentRecord["scope"], { readonly kind: "conversation" }>,
+	at: DocumentPoint,
+	policy: ForkPolicy,
+	childConversationId: ConversationId,
+	copies: ForkDocumentCopy[],
+	copiedAddresses: Set<string>,
+	context: Context,
+): Promise<void> {
+	let cursor: Cursor | undefined;
+	do {
+		const page = await storage.scanDocuments({ scope, at }, SCAN_PAGE_SIZE, cursor, context);
+		for (const source of page.items) {
+			if (source.scope.kind !== "conversation" || source.fork !== policy) continue;
+			const id = await storage.mintId<DocumentId>();
+			const identity = {
+				id,
+				kind: source.kind,
+				...(source.key === undefined ? {} : { key: source.key }),
+				scope: { kind: "conversation" as const, conversationId: childConversationId },
+			};
+			const record: DocumentCreate =
+				source.history === "latest"
+					? { ...identity, history: "latest", fork: source.fork }
+					: { ...identity, history: "rewindable", fork: source.fork };
+			const copyAddress = addressId(record);
+			if (copiedAddresses.has(copyAddress)) {
+				const member = record.key === undefined ? record.kind : `${record.kind}/${record.key}`;
+				throw new Error(`Fork selects multiple source documents for ${member}`);
+			}
+			copiedAddresses.add(copyAddress);
+			copies.push({ record, source: { id: source.id, at } });
+		}
+		cursor = page.next;
+	} while (cursor !== undefined);
+}

third_party/pi/packages/durable/src/session/observation.ts

@@ -0,0 +1,292 @@
+import type {
+	Context,
+	JsonValue,
+	ReplicatedStateSource,
+	ReplicatedStateSourceAttachment,
+	ReplicatedStateSourceFrame,
+} from "@earendil-works/chord";
+import { withoutAbortSignal } from "@earendil-works/chord/context";
+import type { Op } from "@earendil-works/chord/delta";
+import { errorMessage } from "../errors.ts";
+import type { JsonObject, WatchEnd, WatchHandle } from "../types.ts";
+
+export type ObservedDocumentValue = Readonly<JsonObject> | null;
+
+/** Maximum exact committed frames retained behind one unavailable watch listener. */
+const MAX_PENDING_WATCH_FRAMES = 100;
+
+/** Canonical terminal update for a retired document incarnation. */
+export const RETIREMENT_OPERATIONS: readonly Op[] = [["r", null]];
+
+/**
+ * Session-to-Chord bridge owned one-to-one by one attached state: a document, or a conversation view. A `null` value
+ * retires it. @internal
+ */
+export class CommittedStateSource<T = ObservedDocumentValue> implements ReplicatedStateSource<T> {
+	readonly #attachments = new Set<SessionSourceAttachment<T>>();
+	#release: (() => void) | undefined;
+	/** Released on disposal. */
+	#value: T | undefined;
+	#cursor = 0;
+	#retired = false;
+	#closed = false;
+
+	constructor(value: T, release: () => void) {
+		this.#value = value;
+		this.#release = release;
+	}
+
+	attach(): ReplicatedStateSourceAttachment<T> {
+		if (this.#closed) throw new Error("State source is closed");
+		const attachment = new SessionSourceAttachment({ value: this.#value as T, cursor: this.#cursor }, () => {
+			this.#attachments.delete(attachment);
+			if (this.#attachments.size === 0) this.#finishDisposal();
+		});
+		this.#attachments.add(attachment);
+		return attachment;
+	}
+
+	advance(value: T, ops: readonly Op[], context: Context): void {
+		if (this.#closed || this.#retired) return;
+		this.#value = value;
+		this.#cursor += 1;
+		if (value === null) this.#retired = true;
+		const frame: ReplicatedStateSourceFrame<T> = {
+			cursor: this.#cursor,
+			value,
+			ops,
+			context,
+		};
+		for (const attachment of [...this.#attachments]) attachment.publish(frame);
+	}
+
+	/** Stop publishing; the attached states keep their last value. A failure ends them the same way. */
+	closeSession(): void {
+		if (this.#closed) return;
+		for (const attachment of [...this.#attachments]) attachment.dispose();
+		this.#finishDisposal();
+	}
+
+	#finishDisposal(): void {
+		if (this.#closed) return;
+		this.#closed = true;
+		this.#value = undefined;
+		const release = this.#release;
+		this.#release = undefined;
+		release?.();
+	}
+}
+
+class SessionSourceAttachment<T> implements ReplicatedStateSourceAttachment<T> {
+	readonly snapshot: { readonly value: T; readonly cursor: number };
+	#release: (() => void) | undefined;
+	readonly #frames: ReplicatedStateSourceFrame<T>[] = [];
+	#listener: ((frame: ReplicatedStateSourceFrame<T>) => void) | undefined;
+	#activated = false;
+	#disposed = false;
+	#scheduled = false;
+	#delivering = false;
+
+	constructor(snapshot: { readonly value: T; readonly cursor: number }, release: () => void) {
+		this.snapshot = snapshot;
+		this.#release = release;
+	}
+
+	activate(listener: (frame: ReplicatedStateSourceFrame<T>) => void): void {
+		if (this.#activated) throw new Error("State attachment is already active");
+		if (this.#disposed) throw new Error("State attachment is disposed");
+		this.#activated = true;
+		this.#listener = listener;
+		this.#drain();
+	}
+
+	publish(frame: ReplicatedStateSourceFrame<T>): void {
+		if (this.#disposed) return;
+		this.#frames.push(frame);
+		if (!this.#activated || this.#delivering || this.#scheduled) return;
+		this.#scheduled = true;
+		queueMicrotask(() => {
+			this.#scheduled = false;
+			if (this.#disposed) return;
+			try {
+				this.#drain();
+			} catch {
+				// Chord's frame listener contains source-contract failures. Isolate an
+				// unexpected direct listener failure to this attachment as well.
+				this.dispose();
+			}
+		});
+	}
+
+	dispose(): void {
+		if (this.#disposed) return;
+		this.#disposed = true;
+		this.#frames.length = 0;
+		this.#listener = undefined;
+		const release = this.#release;
+		this.#release = undefined;
+		release?.();
+	}
+
+	#drain(): void {
+		const listener = this.#listener;
+		if (listener === undefined || this.#delivering || this.#disposed) return;
+		this.#delivering = true;
+		try {
+			while (!this.#disposed) {
+				const frame = this.#frames.shift();
+				if (frame === undefined) break;
+				listener(frame);
+			}
+		} finally {
+			this.#delivering = false;
+		}
+	}
+}
+
+type WatchFrame<T> = {
+	readonly value: T;
+	readonly ops: readonly Op[];
+	readonly context: Context;
+};
+
+/**
+ * Serialized exact-frame watch bound to one document incarnation or conversation view. A `null` value retires it.
+ * @internal
+ */
+export class CommittedWatch<T = ObservedDocumentValue> implements WatchHandle<T> {
+	readonly #detach: () => void;
+	readonly #report: (error: unknown) => void;
+	readonly #replace: (() => T) | undefined;
+	readonly #closed = Promise.withResolvers<WatchEnd>();
+	readonly #pending: WatchFrame<T>[] = [];
+	#value: T;
+	#listener: ((value: T, ops: readonly Op[], context: Context) => Promise<void>) | undefined;
+	#started = false;
+	#scheduled = false;
+	#running = false;
+	#retired = false;
+	/** Set once, by `#terminate()`. */
+	#end: WatchEnd | undefined;
+	#cancellationSignal: AbortSignal | undefined;
+	#cancellationListener: (() => void) | undefined;
+
+	/**
+	 * `report` gets a listener's throw, which also ends the watch. `replace` gives the value an overflow delivers; by
+	 * default the newest value.
+	 */
+	constructor(value: T, detach: () => void, report: (error: unknown) => void, replace?: () => T) {
+		this.#value = value;
+		this.#detach = detach;
+		this.#report = report;
+		this.#replace = replace;
+	}
+
+	get value(): T {
+		return this.#value;
+	}
+
+	get closed(): Promise<WatchEnd> {
+		return this.#closed.promise;
+	}
+
+	start(listener: (value: T, ops: readonly Op[], context: Context) => Promise<void>): void {
+		if (this.#started) throw new Error("Watch is already started");
+		if (this.#end !== undefined) throw new Error("Watch is stopped");
+		this.#started = true;
+		this.#listener = listener;
+		if (this.#pending.length > 0) this.#schedule();
+	}
+
+	stop(): Promise<WatchEnd> {
+		this.#terminate({ reason: "stopped" });
+		return this.#closed.promise;
+	}
+
+	observeCancellation(signal: AbortSignal): void {
+		if (this.#cancellationSignal !== undefined) throw new Error("Watch cancellation is already installed");
+		if (this.#end !== undefined) return;
+		this.#cancellationSignal = signal;
+		this.#cancellationListener = () => this.cancel();
+		signal.addEventListener("abort", this.#cancellationListener, { once: true });
+		if (signal.aborted) this.cancel();
+	}
+
+	cancel(): void {
+		this.#terminate({ reason: "cancelled" });
+	}
+
+	/** End with `session_closed`, or with `session_failed` and the error when the Session failed. */
+	closeSession(failure?: { readonly error: unknown }): void {
+		this.#terminate(
+			failure === undefined ? { reason: "session_closed" } : { reason: "session_failed", error: failure.error },
+		);
+	}
+
+	advance(value: T, ops: readonly Op[], context: Context): void {
+		if (this.#end !== undefined || this.#retired) return;
+		if (value === null) this.#retired = true;
+		if (this.#pending.length >= MAX_PENDING_WATCH_FRAMES) {
+			this.#pending.length = 0;
+			const replacement = this.#replace?.() ?? value;
+			this.#pending.push({ value: replacement, ops: [["r", replacement as JsonValue]], context });
+		} else {
+			this.#pending.push({ value, ops, context });
+		}
+		if (this.#started) this.#schedule();
+	}
+
+	#schedule(): void {
+		if (this.#scheduled || this.#running || this.#end !== undefined) return;
+		this.#scheduled = true;
+		queueMicrotask(() => {
+			this.#scheduled = false;
+			void this.#drain();
+		});
+	}
+
+	async #drain(): Promise<void> {
+		if (this.#running || this.#end !== undefined || !this.#started) return;
+		this.#running = true;
+		try {
+			while (this.#end === undefined) {
+				const frame = this.#pending.shift();
+				if (frame === undefined) break;
+				this.#value = frame.value;
+				const deliveryContext = withoutAbortSignal(frame.context);
+				try {
+					await this.#listener!(frame.value, frame.ops, deliveryContext);
+				} catch (error) {
+					if (this.#end === undefined) {
+						this.#report(error);
+						this.#terminate({ reason: "listener_error", error: toError(error) });
+					}
+					break;
+				}
+				if (frame.value === null) {
+					this.#terminate({ reason: "retired" });
+					break;
+				}
+			}
+		} finally {
+			this.#running = false;
+			if (this.#end === undefined && this.#pending.length > 0) this.#schedule();
+		}
+	}
+
+	/** End the watch, once: detach, drop pending frames, and settle `closed`. A callback already running keeps running. */
+	#terminate(end: WatchEnd): void {
+		if (this.#end !== undefined) return;
+		this.#end = end;
+		this.#detach();
+		this.#pending.length = 0;
+		if (this.#cancellationListener !== undefined) {
+			this.#cancellationSignal?.removeEventListener("abort", this.#cancellationListener);
+		}
+		this.#closed.resolve(end);
+	}
+}
+
+function toError(error: unknown): Error {
+	return error instanceof Error ? error : new Error(errorMessage(error));
+}

third_party/pi/packages/durable/src/storage/jsonl/index.ts

@@ -0,0 +1,6 @@
+export {
+	JsonlCorruptionError,
+	JsonlStorage,
+	type JsonlStorageOptions,
+	JsonlStoragePoisonedError,
+} from "./storage.ts";

third_party/pi/packages/durable/src/storage/jsonl/node.ts

@@ -0,0 +1,14 @@
+import type { Context } from "@earendil-works/chord";
+import { NodeExecutionEnv } from "../../env/node.ts";
+import { JsonlStorage, type JsonlStorageOptions } from "./storage.ts";
+
+/** Open or create a JSONL storage directory using the local Node filesystem. */
+export async function openNodeJsonlStorage(
+	directory: string,
+	context: Context,
+	options: JsonlStorageOptions = {},
+): Promise<JsonlStorage> {
+	return JsonlStorage.open(directory, new NodeExecutionEnv({ cwd: process.cwd() }), context, options);
+}
+
+export { JsonlStorage, type JsonlStorageOptions } from "./storage.ts";

third_party/pi/packages/durable/src/storage/jsonl/storage.ts

@@ -0,0 +1,846 @@
+import type { Context, JsonValue } from "@earendil-works/chord";
+import type { FileError, FileSystem } from "../../env/index.ts";
+import { StorageRequestError } from "../../errors.ts";
+import { idFromNumber, seqFromNumber } from "../../ids.ts";
+import type {
+	ConversationId,
+	ConversationQuery,
+	Cursor,
+	DocumentAddress,
+	DocumentContent,
+	DocumentCreate,
+	DocumentId,
+	DocumentPoint,
+	DocumentQuery,
+	EntryId,
+	EntryQuery,
+	Id,
+	Seq,
+	Storage,
+	StorageWrite,
+	SubmissionId,
+	SubmissionQuery,
+	TaskId,
+	TaskQuery,
+	TaskRecord,
+} from "../../types.ts";
+import { MemoryStorage } from "../memory.ts";
+
+const FORMAT_VERSION = 1;
+const MAIN_FILE = "main.jsonl";
+const RECLAIM_SUFFIX = ".reclaim";
+const textDecoder = new TextDecoder("utf-8", { fatal: true });
+
+type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
+
+type MainOperation =
+	| Extract<StorageWrite, { readonly type: "conversation" | "entry" | "submission" | "document.retire" }>
+	| { readonly type: "task"; readonly value: StoredTask }
+	| { readonly type: "task.sidecar"; readonly id: TaskId; readonly ordinal: number }
+	| { readonly type: "document.create"; readonly record: DocumentCreate; readonly ordinal: number }
+	| { readonly type: "document.change"; readonly id: DocumentId; readonly ordinal: number };
+
+type MainMarker = {
+	readonly format: typeof FORMAT_VERSION;
+	readonly type: "commit";
+	readonly seq: Seq;
+	readonly writes: readonly MainOperation[];
+};
+
+type SidecarPayload =
+	| { readonly type: "task"; readonly value: StoredTask }
+	| { readonly type: "document"; readonly id: DocumentId; readonly content: DocumentContent };
+
+type SidecarRecord = {
+	readonly format: typeof FORMAT_VERSION;
+	readonly type: "record";
+	readonly seq: Seq;
+	readonly ordinal: number;
+	readonly payload: SidecarPayload;
+};
+
+type ParsedLine<T> = {
+	readonly value: T;
+	readonly start: number;
+};
+
+type ParsedFile<T> = {
+	readonly path: string;
+	readonly lines: readonly ParsedLine<T>[];
+};
+
+type EncodedCommit = {
+	readonly marker: string;
+	readonly sidecars: ReadonlyMap<string, string>;
+};
+
+export type JsonlStorageOptions = {
+	/** Flush every affected sidecar before appending the main marker. Defaults to false. */
+	readonly fsync?: boolean;
+};
+
+export class JsonlCorruptionError extends Error {
+	constructor(message: string, cause?: Error) {
+		super(message, cause === undefined ? undefined : { cause });
+		this.name = "JsonlCorruptionError";
+	}
+}
+
+export class JsonlStoragePoisonedError extends Error {
+	constructor(cause: Error) {
+		super("JSONL storage is poisoned and must be reopened", { cause });
+		this.name = "JsonlStoragePoisonedError";
+	}
+}
+
+const isObject = (value: unknown): value is Record<string, unknown> =>
+	value !== null && typeof value === "object" && !Array.isArray(value);
+
+const isSafeInteger = (value: unknown): value is number => Number.isSafeInteger(value);
+
+const sidecarFileName = (kind: "doc" | "task", id: DocumentId | TaskId): string => `${kind}-${id}.jsonl`;
+
+const isSidecarFileName = (name: string): boolean => /^(?:doc|task)-(?:0|[1-9]\d*)\.jsonl$/.test(name);
+
+const isReclaimFileName = (name: string): boolean => /^(?:doc|task)-(?:0|[1-9]\d*)\.jsonl\.reclaim$/.test(name);
+
+const isCurrentOnly = (record: DocumentCreate): boolean =>
+	record.scope.kind !== "conversation" || record.history === "latest";
+
+const sidecarKey = (file: string, seq: Seq, ordinal: number): string => JSON.stringify([file, seq, ordinal]);
+
+const jsonLine = (value: MainMarker | SidecarRecord): string => `${JSON.stringify(value)}\n`;
+
+const errorFromFile = (action: string, error: FileError): Error =>
+	new Error(`JSONL ${action} failed: ${error.message}`, { cause: error });
+
+const parseJson = (text: string, description: string): unknown => {
+	try {
+		return JSON.parse(text) as unknown;
+	} catch (error) {
+		throw new JsonlCorruptionError(`Malformed complete ${description}`, error instanceof Error ? error : undefined);
+	}
+};
+
+const validateMainOperation = (value: unknown, description: string): MainOperation => {
+	if (!isObject(value) || typeof value.type !== "string") {
+		throw new JsonlCorruptionError(`Invalid write in ${description}`);
+	}
+	switch (value.type) {
+		case "conversation":
+		case "entry":
+		case "submission":
+			if (!isObject(value.value) || !isSafeInteger(value.value.id)) {
+				throw new JsonlCorruptionError(`Invalid ${value.type} write in ${description}`);
+			}
+			return value as MainOperation;
+		case "task":
+			if (
+				!isObject(value.value) ||
+				!isSafeInteger(value.value.id) ||
+				!isObject(value.value.state) ||
+				value.value.state.status !== "terminal"
+			) {
+				throw new JsonlCorruptionError(`Invalid terminal task write in ${description}`);
+			}
+			return value as MainOperation;
+		case "document.retire":
+			if (!isSafeInteger(value.id)) throw new JsonlCorruptionError(`Invalid document retirement in ${description}`);
+			return value as MainOperation;
+		case "task.sidecar":
+			if (!isSafeInteger(value.id) || !isSafeInteger(value.ordinal) || value.ordinal < 0) {
+				throw new JsonlCorruptionError(`Invalid task sidecar write in ${description}`);
+			}
+			return value as MainOperation;
+		case "document.create":
+			if (
+				!isObject(value.record) ||
+				!isSafeInteger(value.record.id) ||
+				!isSafeInteger(value.ordinal) ||
+				value.ordinal < 0
+			) {
+				throw new JsonlCorruptionError(`Invalid document creation in ${description}`);
+			}
+			return value as MainOperation;
+		case "document.change":
+			if (!isSafeInteger(value.id) || !isSafeInteger(value.ordinal) || value.ordinal < 0) {
+				throw new JsonlCorruptionError(`Invalid document change in ${description}`);
+			}
+			return value as MainOperation;
+		default:
+			throw new JsonlCorruptionError(`Unknown write type in ${description}`);
+	}
+};
+
+const parseMainMarker = (text: string, line: number): MainMarker => {
+	const description = `${MAIN_FILE} line ${line}`;
+	const value = parseJson(text, description);
+	if (
+		!isObject(value) ||
+		value.format !== FORMAT_VERSION ||
+		value.type !== "commit" ||
+		!isSafeInteger(value.seq) ||
+		value.seq < 1 ||
+		!Array.isArray(value.writes)
+	) {
+		throw new JsonlCorruptionError(`Invalid commit marker in ${description}`);
+	}
+	return {
+		format: FORMAT_VERSION,
+		type: "commit",
+		seq: seqFromNumber(value.seq),
+		writes: value.writes.map((write) => validateMainOperation(write, description)),
+	};
+};
+
+const validateDocumentContent = (value: unknown, description: string): DocumentContent => {
+	if (!isObject(value) || !isSafeInteger(value.version) || value.version < 1) {
+		throw new JsonlCorruptionError(`Invalid document content in ${description}`);
+	}
+	if (value.kind === "base" && isObject(value.value)) return value as DocumentContent;
+	if (value.kind === "delta" && Array.isArray(value.ops)) return value as DocumentContent;
+	throw new JsonlCorruptionError(`Invalid document content in ${description}`);
+};
+
+const parseSidecarRecord = (text: string, file: string, line: number): SidecarRecord => {
+	const description = `${file} line ${line}`;
+	const value = parseJson(text, description);
+	if (
+		!isObject(value) ||
+		value.format !== FORMAT_VERSION ||
+		value.type !== "record" ||
+		!isSafeInteger(value.seq) ||
+		value.seq < 1 ||
+		!isSafeInteger(value.ordinal) ||
+		value.ordinal < 0 ||
+		!isObject(value.payload) ||
+		typeof value.payload.type !== "string"
+	) {
+		throw new JsonlCorruptionError(`Invalid sidecar record in ${description}`);
+	}
+	if (value.payload.type === "task") {
+		if (
+			!isObject(value.payload.value) ||
+			!isSafeInteger(value.payload.value.id) ||
+			!isObject(value.payload.value.state) ||
+			value.payload.value.state.status === "terminal"
+		) {
+			throw new JsonlCorruptionError(`Invalid live task record in ${description}`);
+		}
+	} else if (value.payload.type === "document") {
+		if (!isSafeInteger(value.payload.id)) {
+			throw new JsonlCorruptionError(`Invalid document record in ${description}`);
+		}
+		validateDocumentContent(value.payload.content, description);
+	} else {
+		throw new JsonlCorruptionError(`Unknown sidecar record type in ${description}`);
+	}
+	return value as SidecarRecord;
+};
+
+/** Portable JSONL implementation of the storage contract. */
+export class JsonlStorage implements Storage {
+	private readonly fs: FileSystem;
+	private readonly directory: string;
+	private readonly mainPath: string;
+	private readonly fsync: boolean;
+	private readonly memory = new MemoryStorage();
+	private readonly currentOnlyDocuments = new Set<DocumentId>();
+	private readonly liveTaskSidecars = new Set<TaskId>();
+	private closed = false;
+	private poisonError: JsonlStoragePoisonedError | undefined;
+
+	private constructor(fs: FileSystem, directory: string, mainPath: string, options: JsonlStorageOptions) {
+		this.fs = fs;
+		this.directory = directory;
+		this.mainPath = mainPath;
+		this.fsync = options.fsync ?? false;
+	}
+
+	/** Open or create a JSONL storage directory using the supplied filesystem. */
+	static async open(
+		directory: string,
+		fs: FileSystem,
+		context: Context,
+		options: JsonlStorageOptions = {},
+	): Promise<JsonlStorage> {
+		const absolute = await fs.absolutePath(directory, context);
+		if (!absolute.ok) throw errorFromFile("path resolution", absolute.error);
+		const created = await fs.createDir(absolute.value, { recursive: true }, context);
+		if (!created.ok) throw errorFromFile("directory creation", created.error);
+		const mainPathResult = await fs.joinPath([absolute.value, MAIN_FILE], context);
+		if (!mainPathResult.ok) throw errorFromFile("path join", mainPathResult.error);
+		const storage = new JsonlStorage(fs, absolute.value, mainPathResult.value, options);
+		await storage.recover(context);
+		return storage;
+	}
+
+	async commit(writes: readonly StorageWrite[], context: Context): Promise<Seq> {
+		this.assertUsable();
+		const prepared = this.memory.prepareCommit(writes);
+		const encoded = this.encodeCommit(prepared.seq, prepared.writes);
+		const reclamations = this.planReclamations(prepared.writes, encoded);
+		const sidecars = await Promise.all(
+			[...encoded.sidecars].map(async ([file, content]) => ({
+				file,
+				content,
+				path: await this.resolveFile(file, context),
+			})),
+		);
+
+		for (const sidecar of sidecars) {
+			const result = await this.fs.appendFile(sidecar.path, sidecar.content, context);
+			if (!result.ok) throw this.poison(errorFromFile(`append to ${sidecar.file}`, result.error));
+		}
+		if (this.fsync) {
+			for (const sidecar of sidecars) {
+				const result = await this.fs.flushFile(sidecar.path, context);
+				if (!result.ok) throw this.poison(errorFromFile(`flush of ${sidecar.file}`, result.error));
+			}
+		}
+		const marker = await this.fs.appendFile(this.mainPath, encoded.marker, context);
+		if (!marker.ok) throw this.poison(errorFromFile(`append to ${MAIN_FILE}`, marker.error));
+		const seq = prepared.apply();
+		this.adoptSidecarState(prepared.writes);
+		await this.reclaimSidecars(reclamations, context);
+		return seq;
+	}
+
+	async mintId<I extends Id<string>>(): Promise<I> {
+		return this.store.mintId<I>();
+	}
+
+	async conversation(id: ConversationId, context: Context) {
+		return this.store.conversation(id, context);
+	}
+
+	async scanConversations(query: ConversationQuery, limit: number, cursor: Cursor | undefined, context: Context) {
+		return this.store.scanConversations(query, limit, cursor, context);
+	}
+
+	entry(id: EntryId, context: Context): ReturnType<Storage["entry"]>;
+	entry(conversationId: ConversationId, id: EntryId, context: Context): ReturnType<Storage["entry"]>;
+	async entry(idOrConversationId: EntryId | ConversationId, idOrContext: EntryId | Context, context?: Context) {
+		if (context === undefined)
+			return this.store.entry(idFromNumber<EntryId>(idOrConversationId), idOrContext as Context);
+		if (typeof idOrContext !== "number") throw new StorageRequestError("Storage.entry() requires an entry ID");
+		return this.store.entry(
+			idFromNumber<ConversationId>(idOrConversationId),
+			idFromNumber<EntryId>(idOrContext),
+			context,
+		);
+	}
+
+	async findLatestHeadMarker(
+		conversationId: ConversationId,
+		atOrBeforeEntryId: EntryId | undefined,
+		context: Context,
+	) {
+		return this.store.findLatestHeadMarker(conversationId, atOrBeforeEntryId, context);
+	}
+
+	async scanEntries(query: EntryQuery, limit: number, cursor: Cursor | undefined, context: Context) {
+		return this.store.scanEntries(query, limit, cursor, context);
+	}
+
+	async task(id: TaskId, context: Context) {
+		return this.store.task(id, context);
+	}
+
+	async scanTasks(query: TaskQuery, limit: number, cursor: Cursor | undefined, context: Context) {
+		return this.store.scanTasks(query, limit, cursor, context);
+	}
+
+	async submission(id: SubmissionId, context: Context) {
+		return this.store.submission(id, context);
+	}
+
+	async scanSubmissions(query: SubmissionQuery, limit: number, cursor: Cursor | undefined, context: Context) {
+		return this.store.scanSubmissions(query, limit, cursor, context);
+	}
+
+	async submissionByRequest(conversationId: ConversationId, requestId: string, context: Context) {
+		return this.store.submissionByRequest(conversationId, requestId, context);
+	}
+
+	async findDocument(address: DocumentAddress, at: DocumentPoint, context: Context) {
+		return this.store.findDocument(address, at, context);
+	}
+
+	async document(id: DocumentId, at: DocumentPoint, context: Context) {
+		return this.store.document(id, at, context);
+	}
+
+	async scanDocuments(query: DocumentQuery, limit: number, cursor: Cursor | undefined, context: Context) {
+		return this.store.scanDocuments(query, limit, cursor, context);
+	}
+
+	async close(context: Context): Promise<void> {
+		if (this.closed) return;
+		this.closed = true;
+		await this.memory.close(context);
+	}
+
+	private encodeCommit(seq: Seq, writes: readonly StorageWrite[]): EncodedCommit {
+		const mainWrites: MainOperation[] = [];
+		const records = new Map<string, SidecarRecord[]>();
+		let nextOrdinal = 0;
+		const addSidecar = (file: string, payload: SidecarPayload): number => {
+			const ordinal = nextOrdinal++;
+			let fileRecords = records.get(file);
+			if (fileRecords === undefined) {
+				fileRecords = [];
+				records.set(file, fileRecords);
+			}
+			fileRecords.push({ format: FORMAT_VERSION, type: "record", seq, ordinal, payload });
+			return ordinal;
+		};
+
+		for (const write of writes) {
+			switch (write.type) {
+				case "conversation":
+				case "entry":
+				case "submission":
+				case "document.retire":
+					mainWrites.push(write);
+					break;
+				case "task":
+					if (write.value.state.status === "terminal") mainWrites.push(write);
+					else {
+						const ordinal = addSidecar(sidecarFileName("task", write.value.id), {
+							type: "task",
+							value: write.value,
+						});
+						mainWrites.push({ type: "task.sidecar", id: write.value.id, ordinal });
+					}
+					break;
+				case "document.create": {
+					const ordinal = addSidecar(sidecarFileName("doc", write.record.id), {
+						type: "document",
+						id: write.record.id,
+						content: write.content,
+					});
+					mainWrites.push({ type: "document.create", record: write.record, ordinal });
+					break;
+				}
+				case "document.change": {
+					const ordinal = addSidecar(sidecarFileName("doc", write.id), {
+						type: "document",
+						id: write.id,
+						content: write.content,
+					});
+					mainWrites.push({ type: "document.change", id: write.id, ordinal });
+					break;
+				}
+			}
+		}
+
+		const sidecars = new Map<string, string>();
+		for (const [file, fileRecords] of records) {
+			sidecars.set(file, fileRecords.map((record) => jsonLine(record)).join(""));
+		}
+		const marker: MainMarker = {
+			format: FORMAT_VERSION,
+			type: "commit",
+			seq,
+			writes: mainWrites,
+		};
+		return { marker: jsonLine(marker), sidecars };
+	}
+
+	private planReclamations(writes: readonly StorageWrite[], encoded: EncodedCommit): ReadonlyMap<string, string> {
+		const createdCurrentOnlyDocuments = new Set<DocumentId>();
+		const retiredDocuments = new Set<DocumentId>();
+		const baseDocuments = new Set<DocumentId>();
+		const finalTasks = new Map<TaskId, StoredTask>();
+		for (const write of writes) {
+			switch (write.type) {
+				case "document.create":
+					if (isCurrentOnly(write.record)) createdCurrentOnlyDocuments.add(write.record.id);
+					break;
+				case "document.change":
+					if (write.content.kind === "base") baseDocuments.add(write.id);
+					break;
+				case "document.retire":
+					retiredDocuments.add(write.id);
+					break;
+				case "task":
+					finalTasks.set(write.value.id, write.value);
+					break;
+			}
+		}
+
+		const replacements = new Map<string, string>();
+		const isCurrentOnlyDocument = (id: DocumentId): boolean =>
+			this.currentOnlyDocuments.has(id) || createdCurrentOnlyDocuments.has(id);
+		for (const id of retiredDocuments) {
+			if (isCurrentOnlyDocument(id)) replacements.set(sidecarFileName("doc", id), "");
+		}
+		for (const id of baseDocuments) {
+			if (!isCurrentOnlyDocument(id) || retiredDocuments.has(id)) continue;
+			const file = sidecarFileName("doc", id);
+			const content = encoded.sidecars.get(file);
+			if (content !== undefined) replacements.set(file, content);
+		}
+		for (const [id, task] of finalTasks) {
+			if (
+				task.state.status === "terminal" &&
+				(this.liveTaskSidecars.has(id) || encoded.sidecars.has(sidecarFileName("task", id)))
+			) {
+				replacements.set(sidecarFileName("task", id), "");
+			}
+		}
+		return replacements;
+	}
+
+	private adoptSidecarState(writes: readonly StorageWrite[]): void {
+		for (const write of writes) {
+			if (write.type === "document.create") {
+				if (isCurrentOnly(write.record)) this.currentOnlyDocuments.add(write.record.id);
+			} else if (write.type === "task") {
+				if (write.value.state.status === "terminal") this.liveTaskSidecars.delete(write.value.id);
+				else this.liveTaskSidecars.add(write.value.id);
+			}
+		}
+	}
+
+	/** The marker already published this state, so reclamation is retryable best-effort maintenance. */
+	private async reclaimSidecars(replacements: ReadonlyMap<string, string>, context: Context): Promise<void> {
+		if (replacements.size === 0) return;
+		if (this.fsync) {
+			const flushed = await this.fs.flushFile(this.mainPath, context);
+			if (!flushed.ok) return;
+		}
+		for (const [file, content] of replacements) await this.replaceSidecar(file, content, context);
+	}
+
+	private async replaceSidecar(file: string, content: string, context: Context): Promise<void> {
+		const path = await this.fs.joinPath([this.directory, file], context);
+		if (!path.ok) return;
+		if (content === "") {
+			await this.fs.remove(path.value, { force: true }, context);
+			return;
+		}
+		const temporaryPath = await this.fs.joinPath([this.directory, `${file}${RECLAIM_SUFFIX}`], context);
+		if (!temporaryPath.ok) return;
+		const written = await this.fs.writeFile(temporaryPath.value, content, context);
+		if (!written.ok) return;
+		if (this.fsync) {
+			const flushed = await this.fs.flushFile(temporaryPath.value, context);
+			if (!flushed.ok) return;
+		}
+		await this.fs.renameFile(temporaryPath.value, path.value, context);
+	}
+
+	private async recover(context: Context): Promise<void> {
+		const fs = this.fs;
+		const directory = this.directory;
+		const memory = this.memory;
+		const main = await JsonlStorage.readLines(fs, this.mainPath, MAIN_FILE, context, (text, line) =>
+			parseMainMarker(text, line),
+		);
+		let previousSeq = 0;
+		for (const marker of main.lines) {
+			if (marker.value.seq <= previousSeq) {
+				throw new JsonlCorruptionError(`Commit sequence does not strictly increase in ${MAIN_FILE}`);
+			}
+			previousSeq = marker.value.seq;
+		}
+
+		const listed = await fs.listDir(directory, context);
+		if (!listed.ok) throw errorFromFile("directory listing", listed.error);
+		for (const info of listed.value) {
+			if (info.kind === "file" && isReclaimFileName(info.name)) {
+				await fs.remove(info.path, { force: true }, context);
+			}
+		}
+		const sidecarFiles = listed.value
+			.filter((info) => info.kind === "file" && isSidecarFileName(info.name))
+			.map((info) => info.name)
+			.sort();
+		const parsedFiles = new Map<string, ParsedFile<SidecarRecord>>();
+		const recordByKey = new Map<string, ParsedLine<SidecarRecord>>();
+		for (const file of sidecarFiles) {
+			const pathResult = await fs.joinPath([directory, file], context);
+			if (!pathResult.ok) throw errorFromFile("path join", pathResult.error);
+			const parsed = await JsonlStorage.readLines(fs, pathResult.value, file, context, (text, line) =>
+				parseSidecarRecord(text, file, line),
+			);
+			parsedFiles.set(file, parsed);
+			let previous: SidecarRecord | undefined;
+			for (const line of parsed.lines) {
+				if (
+					previous !== undefined &&
+					(line.value.seq < previous.seq ||
+						(line.value.seq === previous.seq && line.value.ordinal <= previous.ordinal))
+				) {
+					throw new JsonlCorruptionError(`Sidecar records are out of order in ${file}`);
+				}
+				previous = line.value;
+				recordByKey.set(sidecarKey(file, line.value.seq, line.value.ordinal), line);
+			}
+		}
+
+		const currentOnlyDocuments = new Set<DocumentId>();
+		const retiredDocuments = new Set<DocumentId>();
+		const finalTaskIsLive = new Map<TaskId, boolean>();
+		for (const { value: marker } of main.lines) {
+			for (const operation of marker.writes) {
+				if (operation.type === "document.create") {
+					if (isCurrentOnly(operation.record)) currentOnlyDocuments.add(operation.record.id);
+				} else if (operation.type === "document.retire") {
+					retiredDocuments.add(operation.id);
+				} else if (operation.type === "task") {
+					finalTaskIsLive.set(operation.value.id, false);
+				} else if (operation.type === "task.sidecar") {
+					finalTaskIsLive.set(operation.id, true);
+				}
+			}
+		}
+		const retiredCurrentOnlyDocuments = new Set([...retiredDocuments].filter((id) => currentOnlyDocuments.has(id)));
+
+		const latestBases = new Map<DocumentId, SidecarRecord>();
+		for (const { value: marker } of main.lines) {
+			for (const operation of marker.writes) {
+				if (operation.type !== "document.create" && operation.type !== "document.change") continue;
+				const id = operation.type === "document.create" ? operation.record.id : operation.id;
+				if (!currentOnlyDocuments.has(id)) continue;
+				const record = recordByKey.get(
+					sidecarKey(sidecarFileName("doc", id), marker.seq, operation.ordinal),
+				)?.value;
+				if (
+					record?.payload.type !== "document" ||
+					record.payload.id !== id ||
+					record.payload.content.kind !== "base"
+				) {
+					continue;
+				}
+				const previous = latestBases.get(id);
+				if (
+					previous === undefined ||
+					record.seq > previous.seq ||
+					(record.seq === previous.seq && record.ordinal > previous.ordinal)
+				) {
+					latestBases.set(id, record);
+				}
+			}
+		}
+
+		const isBeforeLatestBase = (id: DocumentId, seq: Seq, ordinal: number): boolean => {
+			const base = latestBases.get(id);
+			return base !== undefined && (seq < base.seq || (seq === base.seq && ordinal < base.ordinal));
+		};
+		const terminalTasks = new Set([...finalTaskIsLive].filter(([, live]) => !live).map(([id]) => id));
+		const confirmed = new Set<string>();
+		for (const line of main.lines) {
+			const marker = line.value;
+			const writes: StorageWrite[] = [];
+			for (const operation of marker.writes) {
+				switch (operation.type) {
+					case "conversation":
+					case "entry":
+					case "submission":
+					case "task":
+					case "document.retire":
+						writes.push(operation);
+						break;
+					case "task.sidecar": {
+						const optional = terminalTasks.has(operation.id);
+						const record = JsonlStorage.confirmRecord(
+							marker,
+							operation.ordinal,
+							sidecarFileName("task", operation.id),
+							recordByKey,
+							confirmed,
+							optional,
+						);
+						if (record !== undefined) {
+							if (record.payload.type !== "task" || record.payload.value.id !== operation.id) {
+								throw new JsonlCorruptionError(
+									`Confirmed task sidecar data does not match commit ${marker.seq}`,
+								);
+							}
+							if (!optional) writes.push({ type: "task", value: record.payload.value });
+						}
+						break;
+					}
+					case "document.create":
+					case "document.change": {
+						const id = operation.type === "document.create" ? operation.record.id : operation.id;
+						const reclaimed =
+							retiredCurrentOnlyDocuments.has(id) || isBeforeLatestBase(id, marker.seq, operation.ordinal);
+						const record = JsonlStorage.confirmRecord(
+							marker,
+							operation.ordinal,
+							sidecarFileName("doc", id),
+							recordByKey,
+							confirmed,
+							reclaimed,
+						);
+						let content: DocumentContent | undefined;
+						if (record !== undefined) {
+							if (record.payload.type !== "document" || record.payload.id !== id) {
+								throw new JsonlCorruptionError(
+									`Confirmed document sidecar data does not match commit ${marker.seq}`,
+								);
+							}
+							content = record.payload.content;
+						}
+						if (operation.type === "document.create") {
+							if (content !== undefined && content.kind !== "base") {
+								throw new JsonlCorruptionError(
+									`Document creation lacks a confirmed base in commit ${marker.seq}`,
+								);
+							}
+							writes.push({
+								type: "document.create",
+								record: operation.record,
+								content: reclaimed || content === undefined ? { kind: "base", version: 1, value: {} } : content,
+							});
+						} else if (!reclaimed && content !== undefined) {
+							writes.push({ type: "document.change", id, content });
+						}
+						break;
+					}
+				}
+			}
+			try {
+				memory.prepareCommit(writes, marker.seq).apply();
+			} catch (error) {
+				throw new JsonlCorruptionError(
+					`Invalid committed state at sequence ${marker.seq}`,
+					error instanceof Error ? error : undefined,
+				);
+			}
+		}
+
+		const reclamations = new Map<string, string>();
+		for (const [file, parsed] of parsedFiles) {
+			let unconfirmedAt: number | undefined;
+			for (const line of parsed.lines) {
+				const key = sidecarKey(file, line.value.seq, line.value.ordinal);
+				if (confirmed.has(key)) {
+					if (unconfirmedAt !== undefined) {
+						throw new JsonlCorruptionError(`Confirmed record follows an unconfirmed tail in ${file}`);
+					}
+				} else if (unconfirmedAt === undefined) {
+					unconfirmedAt = line.start;
+				}
+			}
+			if (unconfirmedAt !== undefined) {
+				const truncated = await fs.truncateFile(parsed.path, unconfirmedAt, context);
+				if (!truncated.ok) throw errorFromFile(`tail truncation of ${file}`, truncated.error);
+			}
+
+			const numericId = Number(file.slice(file.indexOf("-") + 1, -".jsonl".length));
+			const confirmedLines = parsed.lines.filter((line) =>
+				confirmed.has(sidecarKey(file, line.value.seq, line.value.ordinal)),
+			);
+			let retainedLines: readonly ParsedLine<SidecarRecord>[] | undefined;
+			if (file.startsWith("task-") && terminalTasks.has(idFromNumber<TaskId>(numericId))) {
+				retainedLines = [];
+			} else if (file.startsWith("doc-")) {
+				const documentId = idFromNumber<DocumentId>(numericId);
+				if (retiredCurrentOnlyDocuments.has(documentId)) {
+					retainedLines = [];
+				} else if (latestBases.has(documentId)) {
+					retainedLines = confirmedLines.filter(
+						(line) => !isBeforeLatestBase(documentId, line.value.seq, line.value.ordinal),
+					);
+				}
+			}
+			if (
+				retainedLines !== undefined &&
+				(retainedLines.length < confirmedLines.length || retainedLines.length === 0)
+			) {
+				reclamations.set(file, retainedLines.map((line) => jsonLine(line.value)).join(""));
+			}
+		}
+		await this.reclaimSidecars(reclamations, context);
+
+		for (const id of currentOnlyDocuments) this.currentOnlyDocuments.add(id);
+		for (const [id, live] of finalTaskIsLive) {
+			if (live) this.liveTaskSidecars.add(id);
+		}
+	}
+
+	private static confirmRecord(
+		marker: MainMarker,
+		ordinal: number,
+		file: string,
+		recordByKey: ReadonlyMap<string, ParsedLine<SidecarRecord>>,
+		confirmed: Set<string>,
+		optional: boolean,
+	): SidecarRecord | undefined {
+		const key = sidecarKey(file, marker.seq, ordinal);
+		if (confirmed.has(key)) throw new JsonlCorruptionError(`Sidecar record is confirmed more than once`);
+		const line = recordByKey.get(key);
+		if (line === undefined) {
+			if (optional) return undefined;
+			throw new JsonlCorruptionError(`Missing confirmed sidecar record ${file} at sequence ${marker.seq}`);
+		}
+		confirmed.add(key);
+		return line.value;
+	}
+
+	private static async readLines<T>(
+		fs: FileSystem,
+		path: string,
+		name: string,
+		context: Context,
+		parse: (text: string, line: number) => T,
+	): Promise<ParsedFile<T>> {
+		const read = await fs.readBinaryFile(path, context);
+		if (!read.ok) {
+			if (read.error.code === "not_found") return { path, lines: [] };
+			throw errorFromFile(`read of ${name}`, read.error);
+		}
+		const bytes = read.value;
+		let completeSize = bytes.length;
+		if (completeSize > 0 && bytes[completeSize - 1] !== 0x0a) {
+			completeSize = bytes.lastIndexOf(0x0a) + 1;
+			const truncated = await fs.truncateFile(path, completeSize, context);
+			if (!truncated.ok) throw errorFromFile(`torn-line truncation of ${name}`, truncated.error);
+		}
+		const lines: ParsedLine<T>[] = [];
+		let start = 0;
+		let lineNumber = 1;
+		for (let end = 0; end < completeSize; end++) {
+			if (bytes[end] !== 0x0a) continue;
+			let text: string;
+			try {
+				text = textDecoder.decode(bytes.subarray(start, end));
+			} catch (error) {
+				throw new JsonlCorruptionError(
+					`Invalid UTF-8 in complete ${name} line ${lineNumber}`,
+					error instanceof Error ? error : undefined,
+				);
+			}
+			lines.push({ value: parse(text, lineNumber), start });
+			start = end + 1;
+			lineNumber++;
+		}
+		return { path, lines };
+	}
+
+	private async resolveFile(file: string, context: Context): Promise<string> {
+		const path = await this.fs.joinPath([this.directory, file], context);
+		if (!path.ok) throw errorFromFile("path join", path.error);
+		return path.value;
+	}
+
+	private get store(): MemoryStorage {
+		this.assertUsable();
+		return this.memory;
+	}
+
+	private poison(cause: Error): JsonlStoragePoisonedError {
+		this.poisonError ??= new JsonlStoragePoisonedError(cause);
+		return this.poisonError;
+	}
+
+	private assertUsable(): void {
+		if (this.closed) throw new Error("JsonlStorage is closed");
+		if (this.poisonError !== undefined) throw this.poisonError;
+	}
+}

third_party/pi/packages/durable/src/storage/memory.ts

@@ -0,0 +1,864 @@
+import type { Context, JsonValue } from "@earendil-works/chord";
+import { applyImmutableBatches, type Op } from "@earendil-works/chord/delta";
+import { StorageRequestError } from "../errors.ts";
+import { idFromNumber, seqFromNumber } from "../ids.ts";
+import type {
+	ConversationId,
+	ConversationQuery,
+	ConversationRecord,
+	Cursor,
+	DocumentAddress,
+	DocumentContent,
+	DocumentCreate,
+	DocumentId,
+	DocumentPoint,
+	DocumentQuery,
+	DocumentRecord,
+	EntryId,
+	EntryQuery,
+	EntryRecord,
+	Id,
+	JsonObject,
+	Page,
+	ScanOrder,
+	Seq,
+	Storage,
+	StorageWrite,
+	StoredDocument,
+	SubmissionId,
+	SubmissionQuery,
+	SubmissionRecord,
+	TaskId,
+	TaskQuery,
+	TaskRecord,
+} from "../types.ts";
+import { nextCursor, type ScanStart, scanStart } from "./scan.ts";
+
+type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
+type TaskStatus = StoredTask["state"]["status"];
+type SubmissionStatus = SubmissionRecord["status"];
+type TableName = "conversation" | "entry" | "task" | "submission" | "document";
+type DocumentRevision = DocumentContent & { readonly seq: Seq };
+type StoredDocumentState = {
+	record: DocumentRecord;
+	revisions: DocumentRevision[];
+};
+type DocumentAction = {
+	create?: DocumentCreate;
+	content?: DocumentContent;
+	retire: boolean;
+};
+
+function* documentDeltaBatches(
+	id: DocumentId,
+	version: number,
+	revisions: readonly DocumentRevision[],
+	start: number,
+): Generator<readonly Op[]> {
+	for (let index = start; index < revisions.length; index++) {
+		const revision = revisions[index]!;
+		if (revision.kind !== "delta" || revision.version !== version) {
+			throw new Error(`Document ${id} crosses a stored version boundary without a base`);
+		}
+		yield revision.ops;
+	}
+}
+
+type DocumentAddressIndex = {
+	ids: DocumentId[];
+	currentId?: DocumentId;
+};
+
+type State = {
+	recordTypes: Map<Id<string>, TableName>;
+	conversations: Map<ConversationId, ConversationRecord>;
+	conversationIds: ConversationId[];
+	conversationIdsByOwnerConversation: Map<ConversationId, ConversationId[]>;
+	conversationIdsByOwnerTask: Map<TaskId, ConversationId[]>;
+	entries: Map<EntryId, EntryRecord>;
+	entryIds: Map<ConversationId, EntryId[]>;
+	headEntryIds: Map<ConversationId, EntryId[]>;
+	entryCommitSeqs: Map<EntryId, Seq>;
+	tasks: Map<TaskId, StoredTask>;
+	taskIds: TaskId[];
+	taskIdsByStatus: Record<TaskStatus, TaskId[]>;
+	submissions: Map<SubmissionId, SubmissionRecord>;
+	submissionIds: SubmissionId[];
+	submissionIdsByStatus: Record<SubmissionStatus, SubmissionId[]>;
+	submissionIdsByRequest: Map<ConversationId, Map<string, SubmissionId>>;
+	documents: Map<DocumentId, StoredDocumentState>;
+	documentAddresses: Map<string, DocumentAddressIndex>;
+	documentIdsByScope: Map<string, DocumentId[]>;
+};
+
+const clone = <T>(value: T): T => {
+	if (value === null || typeof value !== "object") return value;
+	if (Array.isArray(value)) return value.map((item) => clone(item)) as T;
+	const source = value as Record<string, unknown>;
+	const nullPrototype = Object.getPrototypeOf(value) === null;
+	const result = (nullPrototype ? Object.create(null) : {}) as Record<string, unknown>;
+	for (const key of Object.keys(source)) {
+		const copied = clone(source[key]);
+		if (!nullPrototype && key in result) {
+			Object.defineProperty(result, key, {
+				value: copied,
+				writable: true,
+				enumerable: true,
+				configurable: true,
+			});
+		} else {
+			result[key] = copied;
+		}
+	}
+	return result as T;
+};
+
+const freeze = <T>(value: T): T => {
+	if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
+	for (const child of Object.values(value)) freeze(child);
+	return Object.freeze(value);
+};
+
+const cursorId = <I extends Id<string>>(cursor: Readonly<Record<string, JsonValue>> | undefined): I | undefined => {
+	const after = cursor?.after;
+	if (after === undefined) return undefined;
+	if (typeof after !== "number" || !Number.isSafeInteger(after))
+		throw new StorageRequestError("Invalid storage cursor");
+	return idFromNumber<I>(after);
+};
+
+const lowerBound = <I extends Id<string>>(ids: readonly I[], target: number): number => {
+	let low = 0;
+	let high = ids.length;
+	while (low < high) {
+		const middle = (low + high) >>> 1;
+		if (ids[middle] < target) low = middle + 1;
+		else high = middle;
+	}
+	return low;
+};
+
+const upperBound = <I extends Id<string>>(ids: readonly I[], target: number): number => {
+	let low = 0;
+	let high = ids.length;
+	while (low < high) {
+		const middle = (low + high) >>> 1;
+		if (ids[middle] <= target) low = middle + 1;
+		else high = middle;
+	}
+	return low;
+};
+
+const insertSorted = <I extends Id<string>>(ids: I[], id: I): void => {
+	if (ids.length === 0 || ids[ids.length - 1] < id) ids.push(id);
+	else ids.splice(lowerBound(ids, id), 0, id);
+};
+
+const removeSorted = <I extends Id<string>>(ids: I[], id: I): void => {
+	const index = lowerBound(ids, id);
+	if (ids[index] === id) ids.splice(index, 1);
+};
+
+const insertMapId = <K, I extends Id<string>>(index: Map<K, I[]>, key: K, id: I): void => {
+	let ids = index.get(key);
+	if (ids === undefined) {
+		ids = [];
+		index.set(key, ids);
+	}
+	insertSorted(ids, id);
+};
+
+const scopeKey = (scope: DocumentRecord["scope"]): string => {
+	switch (scope.kind) {
+		case "session":
+			return JSON.stringify(["session"]);
+		case "conversation":
+			return JSON.stringify(["conversation", scope.conversationId]);
+		case "task":
+			return JSON.stringify(["task", scope.taskId]);
+	}
+};
+
+const addressKey = (address: DocumentAddress): string =>
+	JSON.stringify([
+		address.kind,
+		scopeKey(address.scope),
+		address.key === undefined ? ["singleton"] : ["family", address.key],
+	]);
+
+const recordAddressKey = (record: DocumentRecord | DocumentCreate): string =>
+	addressKey({ kind: record.kind, scope: record.scope, key: record.key });
+
+const isAliveAt = (record: DocumentRecord, at: DocumentPoint): boolean => {
+	if (at === "current") return record.retiredAt === undefined;
+	return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt);
+};
+
+const isCurrentOnly = (record: DocumentRecord): boolean =>
+	record.scope.kind !== "conversation" || record.history === "latest";
+
+const tableContaining = (state: State, id: Id<string>): TableName | undefined => state.recordTypes.get(id);
+
+const page = <T extends { readonly id: Id<string> }>(
+	values: readonly T[],
+	limit: number,
+	order: ScanOrder = "ascending",
+): Page<T, Cursor> => {
+	const items = values.slice(0, limit);
+	if (values.length <= limit) return { items: clone(items) };
+	return { items: clone(items), next: nextCursor(items.at(-1)!.id, order) };
+};
+
+/** Indexes of sorted `ids` in scan order, after the cursor's ID when there is one. */
+function* scanIndexes(ids: readonly Id<string>[], start: ScanStart): Generator<number> {
+	if (start.order === "ascending") {
+		const first = start.after === undefined ? 0 : upperBound(ids, start.after);
+		for (let index = first; index < ids.length; index++) yield index;
+	} else {
+		const end = start.after === undefined ? ids.length : lowerBound(ids, start.after);
+		for (let index = end - 1; index >= 0; index--) yield index;
+	}
+}
+
+/** A fully validated, detached state mutation whose application performs no fallible preparation. */
+export interface PreparedMemoryCommit {
+	readonly seq: Seq;
+	/** Deeply frozen detached writes for persistence. */
+	readonly writes: readonly StorageWrite[];
+	apply(): Seq;
+}
+
+/**
+ * Detached in-memory reference implementation of `Storage`.
+ *
+ * Reads and retained writes are cloned intentionally to match the ownership boundary
+ * of serialization-backed stores. This is backend conformance, not validation.
+ */
+export class MemoryStorage implements Storage {
+	private readonly state: State = {
+		recordTypes: new Map(),
+		conversations: new Map(),
+		conversationIds: [],
+		conversationIdsByOwnerConversation: new Map(),
+		conversationIdsByOwnerTask: new Map(),
+		entries: new Map(),
+		entryIds: new Map(),
+		headEntryIds: new Map(),
+		entryCommitSeqs: new Map(),
+		tasks: new Map(),
+		taskIds: [],
+		taskIdsByStatus: { pending: [], running: [], waiting: [], completing: [], terminal: [] },
+		submissions: new Map(),
+		submissionIds: [],
+		submissionIdsByStatus: { queued: [], placed: [], done: [], unanswered: [] },
+		submissionIdsByRequest: new Map(),
+		documents: new Map(),
+		documentAddresses: new Map(),
+		documentIdsByScope: new Map(),
+	};
+	private nextId = 2;
+	private nextSeq = 1;
+	private closed = false;
+
+	async commit(writes: readonly StorageWrite[], _context: Context): Promise<Seq> {
+		return this.prepareCommit(writes).apply();
+	}
+
+	/** Validate and detach one commit without changing observable state. */
+	prepareCommit(writes: readonly StorageWrite[], seq: Seq = seqFromNumber(this.nextSeq)): PreparedMemoryCommit {
+		this.assertOpen();
+		if (!Number.isSafeInteger(seq) || seq < this.nextSeq) {
+			throw new Error(`Commit sequence ${seq} does not strictly increase`);
+		}
+		const detachedWrites = freeze(this.resolveDocumentCopies(writes.map((write) => clone(write))));
+		this.checkGlobalIds(detachedWrites);
+		const documentActions = this.prepareDocumentActions(detachedWrites);
+		this.checkDocumentActions(documentActions);
+		let applied = false;
+		return {
+			seq,
+			writes: detachedWrites,
+			apply: () => {
+				if (!applied) {
+					applied = true;
+					this.applyPreparedCommit(detachedWrites, documentActions, seq);
+				}
+				return seq;
+			},
+		};
+	}
+
+	private resolveDocumentCopies(writes: StorageWrite[]): StorageWrite[] {
+		if (!writes.some((write) => write.type === "document.copy")) return writes;
+		const changedDocumentIds = new Set<DocumentId>();
+		for (const write of writes) {
+			if (write.type === "document.create" || write.type === "document.copy") {
+				changedDocumentIds.add(write.record.id);
+			} else if (write.type === "document.change" || write.type === "document.retire") {
+				changedDocumentIds.add(write.id);
+			}
+		}
+		return writes.map((write) => {
+			if (write.type !== "document.copy") return write;
+			try {
+				if (changedDocumentIds.has(write.source.id)) {
+					throw new Error(`Fork source document ${write.source.id} is changed in the copy batch`);
+				}
+				const stored = this.materializeDocument(write.source.id, write.source.at);
+				if (stored === undefined) throw new Error(`Fork source document ${write.source.id} cannot be read`);
+				if (
+					stored.record.scope.kind !== "conversation" ||
+					write.record.scope.kind !== "conversation" ||
+					stored.record.kind !== write.record.kind ||
+					stored.record.key !== write.record.key ||
+					stored.record.history !== write.record.history ||
+					stored.record.fork !== write.record.fork
+				) {
+					throw new Error(`Fork source document ${write.source.id} does not match the copied record`);
+				}
+				return {
+					type: "document.create",
+					record: write.record,
+					content: { kind: "base", version: stored.version, value: stored.value },
+				};
+			} catch (error) {
+				throw new Error(`Document copy ${write.record.id} was rejected`, { cause: error });
+			}
+		});
+	}
+
+	private applyPreparedCommit(
+		prepared: readonly StorageWrite[],
+		documentActions: ReadonlyMap<DocumentId, DocumentAction>,
+		seq: Seq,
+	): Seq {
+		for (const write of prepared) {
+			switch (write.type) {
+				case "conversation": {
+					this.state.recordTypes.set(write.value.id, "conversation");
+					this.state.conversations.set(write.value.id, write.value);
+					insertSorted(this.state.conversationIds, write.value.id);
+					const owner = write.value.owner;
+					if (owner !== undefined) {
+						insertMapId(this.state.conversationIdsByOwnerConversation, owner.conversationId, write.value.id);
+						insertMapId(this.state.conversationIdsByOwnerTask, owner.taskId, write.value.id);
+					}
+					this.nextId = Math.max(this.nextId, write.value.id + 1);
+					break;
+				}
+				case "entry": {
+					this.state.recordTypes.set(write.value.id, "entry");
+					this.state.entries.set(write.value.id, write.value);
+					this.state.entryCommitSeqs.set(write.value.id, seq);
+					let ids = this.state.entryIds.get(write.value.conversationId);
+					if (ids === undefined) {
+						ids = [];
+						this.state.entryIds.set(write.value.conversationId, ids);
+					}
+					insertSorted(ids, write.value.id);
+					if (write.value.head !== undefined) {
+						let headIds = this.state.headEntryIds.get(write.value.conversationId);
+						if (headIds === undefined) {
+							headIds = [];
+							this.state.headEntryIds.set(write.value.conversationId, headIds);
+						}
+						insertSorted(headIds, write.value.id);
+					}
+					this.nextId = Math.max(this.nextId, write.value.id + 1);
+					break;
+				}
+				case "task": {
+					this.state.recordTypes.set(write.value.id, "task");
+					const previous = this.state.tasks.get(write.value.id);
+					if (previous === undefined) {
+						insertSorted(this.state.taskIds, write.value.id);
+						insertSorted(this.state.taskIdsByStatus[write.value.state.status], write.value.id);
+					} else if (previous.state.status !== write.value.state.status) {
+						removeSorted(this.state.taskIdsByStatus[previous.state.status], write.value.id);
+						insertSorted(this.state.taskIdsByStatus[write.value.state.status], write.value.id);
+					}
+					this.state.tasks.set(write.value.id, write.value);
+					this.nextId = Math.max(this.nextId, write.value.id + 1);
+					break;
+				}
+				case "submission": {
+					this.state.recordTypes.set(write.value.id, "submission");
+					const previous = this.state.submissions.get(write.value.id);
+					if (previous === undefined) {
+						insertSorted(this.state.submissionIds, write.value.id);
+						insertSorted(this.state.submissionIdsByStatus[write.value.status], write.value.id);
+					} else if (previous.status !== write.value.status) {
+						removeSorted(this.state.submissionIdsByStatus[previous.status], write.value.id);
+						insertSorted(this.state.submissionIdsByStatus[write.value.status], write.value.id);
+					}
+					if (previous?.requestId !== undefined) {
+						const previousRequests = this.state.submissionIdsByRequest.get(previous.conversationId);
+						if (previousRequests?.get(previous.requestId) === write.value.id) {
+							previousRequests.delete(previous.requestId);
+							if (previousRequests.size === 0) this.state.submissionIdsByRequest.delete(previous.conversationId);
+						}
+					}
+					this.state.submissions.set(write.value.id, write.value);
+					if (write.value.requestId !== undefined) {
+						let requests = this.state.submissionIdsByRequest.get(write.value.conversationId);
+						if (requests === undefined) {
+							requests = new Map();
+							this.state.submissionIdsByRequest.set(write.value.conversationId, requests);
+						}
+						requests.set(write.value.requestId, write.value.id);
+					}
+					this.nextId = Math.max(this.nextId, write.value.id + 1);
+					break;
+				}
+				case "document.copy":
+					throw new Error("Prepared document copy was not resolved");
+				case "document.create":
+				case "document.change":
+				case "document.retire":
+					break;
+			}
+		}
+
+		this.applyDocumentActions(documentActions, seq);
+		this.nextSeq = seq + 1;
+		return seq;
+	}
+
+	async mintId<I extends Id<string>>(): Promise<I> {
+		this.assertOpen();
+		if (!Number.isSafeInteger(this.nextId)) throw new Error("ID space is exhausted");
+		return idFromNumber<I>(this.nextId++);
+	}
+
+	async conversation(id: ConversationId, _context: Context): Promise<ConversationRecord | undefined> {
+		this.assertOpen();
+		const value = this.state.conversations.get(id);
+		return value === undefined ? undefined : clone(value);
+	}
+
+	async scanConversations(
+		query: ConversationQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<ConversationRecord, Cursor>> {
+		this.assertOpen();
+		const ids =
+			query.ownerTaskId !== undefined
+				? (this.state.conversationIdsByOwnerTask.get(query.ownerTaskId) ?? [])
+				: query.ownerConversationId !== undefined
+					? (this.state.conversationIdsByOwnerConversation.get(query.ownerConversationId) ?? [])
+					: this.state.conversationIds;
+		const start = scanStart(query.order, cursor, "ascending");
+		const values: ConversationRecord[] = [];
+		for (const index of scanIndexes(ids, start)) {
+			if (values.length > limit) break;
+			const value = this.state.conversations.get(ids[index]!)!;
+			if (query.ownerConversationId !== undefined && value.owner?.conversationId !== query.ownerConversationId) {
+				continue;
+			}
+			values.push(value);
+		}
+		return page(values, limit, start.order);
+	}
+
+	entry(id: EntryId, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
+	entry(
+		conversationId: ConversationId,
+		id: EntryId,
+		context: Context,
+	): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
+	async entry(
+		idOrConversationId: EntryId | ConversationId,
+		idOrContext: EntryId | Context,
+		context?: Context,
+	): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined> {
+		this.assertOpen();
+		if (context === undefined) {
+			const id = idFromNumber<EntryId>(idOrConversationId);
+			const entry = this.state.entries.get(id);
+			if (entry === undefined) return undefined;
+			return { entry: clone(entry), commitSeq: this.state.entryCommitSeqs.get(id)! };
+		}
+		if (typeof idOrContext !== "number") throw new StorageRequestError("Storage.entry() requires an entry ID");
+		const conversationId = idFromNumber<ConversationId>(idOrConversationId);
+		const id = idFromNumber<EntryId>(idOrContext);
+		const entry = this.visibleEntries(conversationId, id, id).next().value;
+		if (entry === undefined) return undefined;
+		return { entry: clone(entry), commitSeq: this.state.entryCommitSeqs.get(id)! };
+	}
+
+	async findLatestHeadMarker(
+		conversationId: ConversationId,
+		atOrBeforeEntryId: EntryId | undefined,
+		_context: Context,
+	): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> {
+		this.assertOpen();
+		if (!this.state.conversations.has(conversationId)) {
+			throw new StorageRequestError(`Unknown conversation: ${conversationId}`);
+		}
+		let currentId = conversationId;
+		let upperEntryId = atOrBeforeEntryId ?? Number.POSITIVE_INFINITY;
+		while (true) {
+			const ids = this.state.headEntryIds.get(currentId) ?? [];
+			const index = upperBound(ids, upperEntryId) - 1;
+			if (index >= 0) {
+				const entry = this.state.entries.get(ids[index])!;
+				return clone({ ...entry, head: entry.head! });
+			}
+			const conversation = this.state.conversations.get(currentId)!;
+			if (conversation.parent === undefined) return undefined;
+			upperEntryId = Math.min(upperEntryId, conversation.parent.at);
+			currentId = conversation.parent.conversationId;
+		}
+	}
+
+	async scanEntries(
+		query: EntryQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<EntryRecord, Cursor>> {
+		this.assertOpen();
+		const { order, after } = scanStart(query.order, cursor, "descending");
+		// The cursor narrows the bound on the side the scan moves away from.
+		let minEntryId: number | undefined = query.minEntryId;
+		let maxEntryId: number | undefined = query.maxEntryId;
+		if (after !== undefined && order === "descending") maxEntryId = Math.min(maxEntryId ?? after, after - 1);
+		if (after !== undefined && order === "ascending") minEntryId = Math.max(minEntryId ?? after, after + 1);
+		const visible: EntryRecord[] = [];
+		const entries =
+			order === "descending"
+				? this.visibleEntries(query.conversationId, minEntryId, maxEntryId)
+				: this.visibleEntriesAscending(query.conversationId, minEntryId, maxEntryId);
+		for (const entry of entries) {
+			visible.push(entry);
+			if (visible.length > limit) break;
+		}
+		return page(visible, limit, order);
+	}
+
+	async task(id: TaskId, _context: Context): Promise<StoredTask | undefined> {
+		this.assertOpen();
+		const value = this.state.tasks.get(id);
+		return value === undefined ? undefined : clone(value);
+	}
+
+	async scanTasks(
+		query: TaskQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<StoredTask, Cursor>> {
+		this.assertOpen();
+		const start = scanStart(query.order, cursor, "ascending");
+		const ids = query.status === undefined ? this.state.taskIds : this.state.taskIdsByStatus[query.status];
+		const values: StoredTask[] = [];
+		for (const index of scanIndexes(ids, start)) {
+			if (values.length > limit) break;
+			const value = this.state.tasks.get(ids[index]!)!;
+			if (query.conversationId !== undefined && value.conversationId !== query.conversationId) continue;
+			if (query.kind !== undefined && value.kind !== query.kind) continue;
+			if (query.abortRequested !== undefined && value.abortRequested !== query.abortRequested) continue;
+			if (query.background !== undefined && value.background !== query.background) continue;
+			values.push(value);
+		}
+		return page(values, limit, start.order);
+	}
+
+	async submission(id: SubmissionId, _context: Context): Promise<SubmissionRecord | undefined> {
+		this.assertOpen();
+		const value = this.state.submissions.get(id);
+		return value === undefined ? undefined : clone(value);
+	}
+
+	async scanSubmissions(
+		query: SubmissionQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<SubmissionRecord, Cursor>> {
+		this.assertOpen();
+		const start = scanStart(query.order, cursor, "ascending");
+		const ids =
+			query.status === undefined ? this.state.submissionIds : this.state.submissionIdsByStatus[query.status];
+		const values: SubmissionRecord[] = [];
+		for (const index of scanIndexes(ids, start)) {
+			if (values.length > limit) break;
+			const value = this.state.submissions.get(ids[index]!)!;
+			if (query.conversationId !== undefined && value.conversationId !== query.conversationId) continue;
+			values.push(value);
+		}
+		return page(values, limit, start.order);
+	}
+
+	async submissionByRequest(
+		conversationId: ConversationId,
+		requestId: string,
+		_context: Context,
+	): Promise<SubmissionRecord | undefined> {
+		this.assertOpen();
+		const id = this.state.submissionIdsByRequest.get(conversationId)?.get(requestId);
+		if (id === undefined) return undefined;
+		return clone(this.state.submissions.get(id)!);
+	}
+
+	async findDocument(
+		address: DocumentAddress,
+		at: DocumentPoint,
+		_context: Context,
+	): Promise<DocumentRecord | undefined> {
+		this.assertOpen();
+		const index = this.state.documentAddresses.get(addressKey(address));
+		if (at === "current") {
+			return index?.currentId === undefined ? undefined : clone(this.state.documents.get(index.currentId)!.record);
+		}
+		for (const id of index?.ids ?? []) {
+			const record = this.state.documents.get(id)!.record;
+			if (isAliveAt(record, at)) return clone(record);
+		}
+		return undefined;
+	}
+
+	async document(id: DocumentId, at: DocumentPoint, _context: Context): Promise<StoredDocument | undefined> {
+		this.assertOpen();
+		const stored = this.materializeDocument(id, at);
+		return stored === undefined
+			? undefined
+			: {
+					record: clone(stored.record),
+					version: stored.version,
+					value: clone(stored.value),
+					deltasSinceBase: stored.deltasSinceBase,
+				};
+	}
+
+	async scanDocuments(
+		query: DocumentQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<DocumentRecord, Cursor>> {
+		this.assertOpen();
+		const ids = this.state.documentIdsByScope.get(scopeKey(query.scope)) ?? [];
+		const after = cursorId(cursor);
+		const start = after === undefined ? 0 : upperBound(ids, after);
+		const values: DocumentRecord[] = [];
+		for (let index = start; index < ids.length && values.length <= limit; index++) {
+			const record = this.state.documents.get(ids[index]!)!.record;
+			if (query.kind !== undefined && record.kind !== query.kind) continue;
+			if (isAliveAt(record, query.at)) values.push(record);
+		}
+		return page(values, limit);
+	}
+
+	async close(_context: Context): Promise<void> {
+		this.closed = true;
+	}
+
+	private materializeDocument(id: DocumentId, at: DocumentPoint): StoredDocument | undefined {
+		const stored = this.state.documents.get(id);
+		if (stored === undefined) return undefined;
+		if (at !== "current" && isCurrentOnly(stored.record)) {
+			throw new StorageRequestError(`Document ${id} does not retain historical content`);
+		}
+		if (!isAliveAt(stored.record, at)) return undefined;
+		const revisions = at === "current" ? stored.revisions : stored.revisions.filter((revision) => revision.seq <= at);
+		let baseIndex = revisions.length - 1;
+		while (baseIndex >= 0 && revisions[baseIndex]!.kind !== "base") baseIndex--;
+		const base = revisions[baseIndex];
+		if (base?.kind !== "base") throw new Error(`Document ${id} is missing a required base`);
+		const value = applyImmutableBatches(
+			base.value,
+			documentDeltaBatches(id, base.version, revisions, baseIndex + 1),
+		) as JsonObject;
+		return { record: stored.record, version: base.version, value, deltasSinceBase: revisions.length - baseIndex - 1 };
+	}
+
+	private *visibleEntries(
+		conversationId: ConversationId,
+		minEntryId: number = Number.NEGATIVE_INFINITY,
+		maxEntryId: number = Number.POSITIVE_INFINITY,
+	): Generator<EntryRecord> {
+		if (!this.state.conversations.has(conversationId)) {
+			throw new StorageRequestError(`Unknown conversation: ${conversationId}`);
+		}
+		let currentId = conversationId;
+		let upperEntryId = maxEntryId;
+		while (true) {
+			const ids = this.state.entryIds.get(currentId) ?? [];
+			for (let index = upperBound(ids, upperEntryId) - 1; index >= 0; index--) {
+				const id = ids[index];
+				if (id < minEntryId) break;
+				yield this.state.entries.get(id)!;
+			}
+			const conversation = this.state.conversations.get(currentId)!;
+			if (conversation.parent === undefined) break;
+			upperEntryId = Math.min(upperEntryId, conversation.parent.at);
+			if (upperEntryId < minEntryId) break;
+			currentId = conversation.parent.conversationId;
+		}
+	}
+
+	/** Visible entries oldest first: the fork chain's segments from the root conversation forward. */
+	private *visibleEntriesAscending(
+		conversationId: ConversationId,
+		minEntryId: number = Number.NEGATIVE_INFINITY,
+		maxEntryId: number = Number.POSITIVE_INFINITY,
+	): Generator<EntryRecord> {
+		if (!this.state.conversations.has(conversationId)) {
+			throw new StorageRequestError(`Unknown conversation: ${conversationId}`);
+		}
+		const segments: { readonly conversationId: ConversationId; readonly upper: number }[] = [];
+		let currentId = conversationId;
+		let upperEntryId = maxEntryId;
+		while (true) {
+			segments.push({ conversationId: currentId, upper: upperEntryId });
+			const conversation = this.state.conversations.get(currentId)!;
+			if (conversation.parent === undefined) break;
+			upperEntryId = Math.min(upperEntryId, conversation.parent.at);
+			if (upperEntryId < minEntryId) break;
+			currentId = conversation.parent.conversationId;
+		}
+		for (const segment of segments.reverse()) {
+			const ids = this.state.entryIds.get(segment.conversationId) ?? [];
+			for (let index = lowerBound(ids, minEntryId); index < ids.length; index++) {
+				const id = ids[index]!;
+				if (id > segment.upper) break;
+				yield this.state.entries.get(id)!;
+			}
+		}
+	}
+
+	private checkGlobalIds(writes: readonly StorageWrite[]): void {
+		const claimed = new Map<Id<string>, TableName>();
+		for (const write of writes) {
+			if (write.type === "document.change" || write.type === "document.retire") continue;
+			const document = write.type === "document.create" || write.type === "document.copy";
+			const table: TableName = document ? "document" : write.type;
+			const id = document ? write.record.id : write.value.id;
+			const existing = tableContaining(this.state, id);
+			const earlier = claimed.get(id);
+			if (table === "conversation" || table === "entry" || table === "document") {
+				if (existing !== undefined) throw new Error(`ID ${id} already belongs to ${existing}`);
+				if (earlier !== undefined) throw new Error(`ID ${id} is written more than once`);
+			} else {
+				if (existing !== undefined && existing !== table) {
+					throw new Error(`ID ${id} already belongs to ${existing}`);
+				}
+				if (earlier !== undefined && earlier !== table) throw new Error(`ID ${id} is written as two record types`);
+			}
+			claimed.set(id, table);
+		}
+	}
+
+	private prepareDocumentActions(writes: readonly StorageWrite[]): Map<DocumentId, DocumentAction> {
+		const actions = new Map<DocumentId, DocumentAction>();
+		for (const write of writes) {
+			if (write.type !== "document.create" && write.type !== "document.change" && write.type !== "document.retire") {
+				continue;
+			}
+			const id = write.type === "document.create" ? write.record.id : write.id;
+			let action = actions.get(id);
+			if (action === undefined) {
+				action = { retire: false };
+				actions.set(id, action);
+			}
+			switch (write.type) {
+				case "document.create":
+					if (action.create !== undefined || action.content !== undefined) {
+						throw new Error(`Document ${id} has more than one content command`);
+					}
+					action.create = write.record;
+					action.content = write.content;
+					break;
+				case "document.change":
+					if (action.content !== undefined) throw new Error(`Document ${id} has more than one content command`);
+					action.content = write.content;
+					break;
+				case "document.retire":
+					if (action.retire) throw new Error(`Document ${id} is retired more than once`);
+					action.retire = true;
+					break;
+			}
+		}
+		return actions;
+	}
+
+	private checkDocumentActions(actions: ReadonlyMap<DocumentId, DocumentAction>): void {
+		const liveCounts = new Map<string, number>();
+		for (const [id, action] of actions) {
+			const existing = this.state.documents.get(id);
+			if (action.create === undefined && existing === undefined) throw new Error(`Unknown document: ${id}`);
+			if (action.create !== undefined && existing !== undefined) throw new Error(`Document ${id} already exists`);
+			if (existing?.record.retiredAt !== undefined) throw new Error(`Document ${id} is retired`);
+			const previous = existing?.revisions.at(-1);
+			if (action.content?.kind === "delta") {
+				if (previous === undefined) throw new Error(`Document ${id} delta has no base`);
+				if (previous.version !== action.content.version) {
+					throw new Error(`Document ${id} version transition requires a base`);
+				}
+			}
+
+			const key = action.create === undefined ? recordAddressKey(existing!.record) : recordAddressKey(action.create);
+			const currentId = this.state.documentAddresses.get(key)?.currentId;
+			let live = liveCounts.get(key);
+			if (live === undefined) live = currentId === undefined ? 0 : 1;
+			if (action.retire && currentId === id) live--;
+			if (action.create !== undefined && !action.retire) live++;
+			liveCounts.set(key, live);
+		}
+
+		for (const live of liveCounts.values()) {
+			if (live > 1) throw new Error(`Document address already has a current incarnation`);
+		}
+	}
+
+	private applyDocumentActions(actions: ReadonlyMap<DocumentId, DocumentAction>, seq: Seq): void {
+		for (const [id, action] of actions) {
+			let stored = this.state.documents.get(id);
+			if (action.create !== undefined) {
+				const record: DocumentRecord = {
+					...action.create,
+					createdAt: seq,
+					...(action.retire ? { retiredAt: seq } : {}),
+				};
+				stored = { record, revisions: [{ ...action.content!, seq }] };
+				this.state.recordTypes.set(id, "document");
+				this.state.documents.set(id, stored);
+
+				const key = recordAddressKey(record);
+				let address = this.state.documentAddresses.get(key);
+				if (address === undefined) {
+					address = { ids: [] };
+					this.state.documentAddresses.set(key, address);
+				}
+				insertSorted(address.ids, id);
+
+				let scopeIds = this.state.documentIdsByScope.get(scopeKey(record.scope));
+				if (scopeIds === undefined) {
+					scopeIds = [];
+					this.state.documentIdsByScope.set(scopeKey(record.scope), scopeIds);
+				}
+				insertSorted(scopeIds, id);
+				this.nextId = Math.max(this.nextId, id + 1);
+			} else if (action.content !== undefined) {
+				const revision = { ...action.content, seq } as DocumentRevision;
+				if (revision.kind === "base" && isCurrentOnly(stored!.record)) stored!.revisions = [revision];
+				else stored!.revisions.push(revision);
+			}
+
+			if (action.retire && action.create === undefined) stored!.record = { ...stored!.record, retiredAt: seq };
+			if (action.retire && isCurrentOnly(stored!.record)) stored!.revisions = [];
+			if (action.create !== undefined || action.retire) {
+				const address = this.state.documentAddresses.get(recordAddressKey(stored!.record))!;
+				if (action.retire && address.currentId === id) delete address.currentId;
+				if (action.create !== undefined && !action.retire) address.currentId = id;
+			}
+		}
+	}
+
+	private assertOpen(): void {
+		if (this.closed) throw new Error("MemoryStorage is closed");
+	}
+}

third_party/pi/packages/durable/src/storage/scan.ts

@@ -0,0 +1,40 @@
+import { StorageRequestError } from "../errors.ts";
+import type { Id, ScanOrder } from "../types.ts";
+
+/** Where a built-in storage scan starts: its order and the last ID a previous page returned. */
+export type ScanStart = {
+	readonly order: ScanOrder;
+	readonly after: number | undefined;
+};
+
+/**
+ * Resolve a scan's order and position. A cursor continues in the order it was created with, whether the query repeats
+ * that order or omits it; a different `order` throws. A cursor without an order, written before scans had one,
+ * continues in the scan's default order.
+ */
+export function scanStart(
+	requested: ScanOrder | undefined,
+	cursor: Readonly<Record<string, unknown>> | undefined,
+	fallback: ScanOrder,
+): ScanStart {
+	if (requested !== undefined && requested !== "ascending" && requested !== "descending") {
+		throw new StorageRequestError(`Invalid scan order: ${String(requested)}`);
+	}
+	if (cursor === undefined) return { order: requested ?? fallback, after: undefined };
+	const { after, order: stored } = cursor;
+	if (typeof after !== "number" || !Number.isSafeInteger(after))
+		throw new StorageRequestError("Invalid storage cursor");
+	if (stored !== undefined && stored !== "ascending" && stored !== "descending") {
+		throw new StorageRequestError("Invalid storage cursor");
+	}
+	const order = stored ?? fallback;
+	if (requested !== undefined && requested !== order) {
+		throw new StorageRequestError(`The cursor continues a ${order} scan; the query asks for ${requested}`);
+	}
+	return { order, after };
+}
+
+/** The continuation of a scan whose last returned item has `id`. */
+export function nextCursor(id: Id<string>, order: ScanOrder): { readonly after: number; readonly order: ScanOrder } {
+	return { after: id, order };
+}

third_party/pi/packages/durable/src/storage/sqlite/cloudflare.ts

@@ -0,0 +1,192 @@
+import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "./database.ts";
+import { SqliteStorage, type SqliteStorageOptions } from "./storage.ts";
+
+/** Values a Durable Object SQL binding accepts and returns. */
+type DurableObjectSqlValue = ArrayBuffer | string | number | null;
+
+/**
+ * The parts of a SQLite-backed Durable Object's `ctx.storage` this adapter uses, typed structurally so the package
+ * needs no Workers type dependency.
+ */
+export type DurableObjectSqliteStorage = {
+	readonly sql: {
+		exec(
+			query: string,
+			...bindings: DurableObjectSqlValue[]
+		): {
+			next():
+				| { readonly done?: false; readonly value: Record<string, DurableObjectSqlValue> }
+				| { readonly done: true; readonly value?: undefined };
+			toArray(): Record<string, DurableObjectSqlValue>[];
+		};
+	};
+	transaction<T>(closure: () => Promise<T>): Promise<T>;
+};
+
+/** Runs synchronous operations immediately when idle and holds queued calls behind a transaction. */
+class SerialQueue {
+	private tail: Promise<void> = Promise.resolve();
+	private pending = 0;
+
+	run<T>(operation: () => T | Promise<T>): Promise<T> {
+		if (this.pending > 0) return this.enqueue(operation);
+		try {
+			return Promise.resolve(operation());
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+	hold<T>(operation: () => Promise<T>): Promise<T> {
+		return this.enqueue(operation);
+	}
+
+	private enqueue<T>(operation: () => T | Promise<T>): Promise<T> {
+		this.pending++;
+		const result = this.tail.then(operation).finally(() => {
+			this.pending--;
+		});
+		this.tail = result.then(
+			() => {},
+			() => {},
+		);
+		return result;
+	}
+}
+
+const storageQueues = new WeakMap<DurableObjectSqliteStorage, SerialQueue>();
+
+function queueFor(storage: DurableObjectSqliteStorage): SerialQueue {
+	let queue = storageQueues.get(storage);
+	if (queue === undefined) {
+		queue = new SerialQueue();
+		storageQueues.set(storage, queue);
+	}
+	return queue;
+}
+
+/** Durable Object SQL binds numbers as doubles: a `bigint` outside the safe integer range would lose precision. */
+function bindValue(value: SqliteValue): DurableObjectSqlValue {
+	if (value instanceof Uint8Array) return value.slice().buffer as ArrayBuffer;
+	if (typeof value !== "bigint") return value;
+	if (value < BigInt(Number.MIN_SAFE_INTEGER) || value > BigInt(Number.MAX_SAFE_INTEGER)) {
+		throw new RangeError(`Durable Object SQL cannot bind ${value} without losing precision`);
+	}
+	return Number(value);
+}
+
+const bind = (params: readonly SqliteValue[]): DurableObjectSqlValue[] => params.map(bindValue);
+
+/** A row as returned, with `BLOB` columns converted to `Uint8Array` in place; no other copy is made. */
+function row<T>(row: Record<string, DurableObjectSqlValue>): T {
+	for (const key in row) {
+		const value = row[key];
+		if (value instanceof ArrayBuffer) (row as Record<string, unknown>)[key] = new Uint8Array(value);
+	}
+	return row as T;
+}
+
+/** Executes SQL through `ctx.storage.sql`, which has no separate prepare step. */
+class DurableObjectSqliteExecutor implements SqliteExecutor {
+	protected readonly storage: DurableObjectSqliteStorage;
+
+	constructor(storage: DurableObjectSqliteStorage) {
+		this.storage = storage;
+	}
+
+	async exec(sql: string): Promise<void> {
+		this.check();
+		this.storage.sql.exec(sql);
+	}
+
+	async run(sql: string, ...params: SqliteValue[]): Promise<void> {
+		this.check();
+		this.storage.sql.exec(sql, ...bind(params));
+	}
+
+	async get<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T | undefined> {
+		this.check();
+		const result = this.storage.sql.exec(sql, ...bind(params)).next();
+		return result.done ? undefined : row<T>(result.value);
+	}
+
+	async all<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T[]> {
+		this.check();
+		return this.storage.sql
+			.exec(sql, ...bind(params))
+			.toArray()
+			.map((value) => row<T>(value));
+	}
+
+	/** Throws when this handle may no longer run SQL. */
+	protected check(): void {}
+}
+
+/** The handle a transaction callback receives; invalid once the callback settles. */
+class DurableObjectSqliteTransaction extends DurableObjectSqliteExecutor {
+	active = true;
+
+	protected override check(): void {
+		if (!this.active) throw new Error("SQLite transaction handle is no longer active");
+	}
+}
+
+/**
+ * `SqliteDatabase` adapter for the SQLite storage of a Cloudflare Durable Object. Adapters given the same storage
+ * object share a queue; direct calls to the underlying storage bypass it. Integers are JavaScript numbers; binding a
+ * `bigint` outside the safe integer range throws.
+ */
+export class DurableObjectSqliteDatabase extends DurableObjectSqliteExecutor implements SqliteDatabase {
+	private readonly queue: SerialQueue;
+
+	constructor(storage: DurableObjectSqliteStorage) {
+		super(storage);
+		this.queue = queueFor(storage);
+	}
+
+	override exec(sql: string): Promise<void> {
+		return this.queue.run(() => super.exec(sql));
+	}
+
+	override run(sql: string, ...params: SqliteValue[]): Promise<void> {
+		return this.queue.run(() => super.run(sql, ...params));
+	}
+
+	override get<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T | undefined> {
+		return this.queue.run(() => super.get<T>(sql, ...params));
+	}
+
+	override all<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T[]> {
+		return this.queue.run(() => super.all<T>(sql, ...params));
+	}
+
+	/** `ctx.storage.transaction()` commits when the callback resolves and rolls back when it rejects. */
+	transaction<T>(callback: (transaction: SqliteExecutor) => Promise<T>): Promise<T> {
+		return this.queue.hold(() =>
+			this.storage.transaction(async () => {
+				const transaction = new DurableObjectSqliteTransaction(this.storage);
+				try {
+					return await callback(transaction);
+				} finally {
+					transaction.active = false;
+				}
+			}),
+		);
+	}
+
+	/** Waits for queued work. The Durable Object owns its storage, so nothing else closes. */
+	close(): Promise<void> {
+		return this.queue.run(() => {});
+	}
+}
+
+/**
+ * Open durable storage on a SQLite-backed Durable Object, optionally under a table prefix. Calls given the same
+ * storage object share an operation queue.
+ */
+export function openDurableObjectSqliteStorage(
+	storage: DurableObjectSqliteStorage,
+	options: SqliteStorageOptions = {},
+): Promise<SqliteStorage> {
+	return SqliteStorage.open(new DurableObjectSqliteDatabase(storage), options);
+}

third_party/pi/packages/durable/src/storage/sqlite/database.ts

@@ -0,0 +1,38 @@
+/** Values supported by the portable SQLite storage core. */
+export type SqliteValue = null | number | bigint | string | Uint8Array;
+
+/**
+ * Asynchronous SQL operations shared by a database and its transaction handles.
+ *
+ * `exec` runs SQL text without bindings and may contain several statements. `run`, `get`, and `all`
+ * execute one statement with positional bindings. Adapters may cache prepared statements by SQL text,
+ * so callers pass values as bindings instead of interpolating them.
+ */
+export interface SqliteExecutor {
+	exec(sql: string): Promise<void>;
+	run(sql: string, ...params: SqliteValue[]): Promise<void>;
+	get<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T | undefined>;
+	all<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T[]>;
+}
+
+/**
+ * Minimal database facade required by `SqliteStorage`.
+ *
+ * All operations are asynchronous so adapters may execute outside the harness runtime.
+ *
+ * `transaction` passes the callback a transaction handle. All work in the transaction
+ * must use that handle; the handle is invalid after the callback settles. Adapters must
+ * queue unrelated operations and other transactions until the transaction finishes. The
+ * returned promise settles after commit or rollback. Calling the database itself (including
+ * `transaction` or `close`) from inside a callback therefore waits for that transaction and
+ * never settles.
+ *
+ * When the callback rejects, the adapter must roll the transaction back before rejecting
+ * with that same error. If rollback fails, it must reject with a different error (for
+ * example an `AggregateError`) so callers cannot mistake the callback error for a
+ * guaranteed rollback.
+ */
+export interface SqliteDatabase extends SqliteExecutor {
+	transaction<T>(callback: (transaction: SqliteExecutor) => Promise<T>): Promise<T>;
+	close(): Promise<void>;
+}

third_party/pi/packages/durable/src/storage/sqlite/index.ts

@@ -0,0 +1,8 @@
+export type { SqliteDatabase, SqliteExecutor, SqliteValue } from "./database.ts";
+export {
+	applySqliteMigrations,
+	CURRENT_SQLITE_SCHEMA_VERSION,
+	SQLITE_MIGRATIONS,
+	type SqliteMigration,
+} from "./migrations.ts";
+export { SqliteStorage, type SqliteStorageOptions } from "./storage.ts";

third_party/pi/packages/durable/src/storage/sqlite/migrations.ts

@@ -0,0 +1,140 @@
+import type { SqliteDatabase } from "./database.ts";
+import { DEFAULT_SQLITE_SCHEMA, type SqliteSchema } from "./schema.ts";
+
+export type SqliteMigration = {
+	readonly version: number;
+	readonly statements: readonly string[];
+};
+
+/** Build immutable, ordered schema history for a set of SQLite identifiers. */
+export function createSqliteMigrations(schema: SqliteSchema): readonly SqliteMigration[] {
+	const { tables, indexes } = schema;
+	// next_id is TEXT because node:sqlite rejects INTEGER results outside JavaScript's safe integer range.
+	const initialSchema: readonly string[] = [
+		`CREATE TABLE ${tables.durableMetadata} (
+			singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
+			next_id TEXT NOT NULL,
+			next_seq INTEGER NOT NULL
+		) STRICT`,
+		`INSERT INTO ${tables.durableMetadata} (singleton, next_id, next_seq) VALUES (1, '2', 1)`,
+		`CREATE TABLE ${tables.recordIds} (
+			id INTEGER PRIMARY KEY,
+			record_type TEXT NOT NULL CHECK (record_type IN ('conversation', 'entry', 'task', 'submission', 'document'))
+		) STRICT`,
+		`CREATE TABLE ${tables.conversations} (
+			id INTEGER PRIMARY KEY,
+			owner_conversation_id INTEGER,
+			owner_task_id INTEGER,
+			record TEXT NOT NULL CHECK (json_valid(record))
+		) STRICT`,
+		`CREATE INDEX ${indexes.conversationsByOwnerConversation} ON ${tables.conversations} (owner_conversation_id, id)`,
+		`CREATE INDEX ${indexes.conversationsByOwnerTask} ON ${tables.conversations} (owner_task_id, id)`,
+		`CREATE TABLE ${tables.entries} (
+			id INTEGER PRIMARY KEY,
+			conversation_id INTEGER NOT NULL,
+			head INTEGER,
+			commit_seq INTEGER NOT NULL,
+			record TEXT NOT NULL CHECK (json_valid(record))
+		) STRICT`,
+		`CREATE INDEX ${indexes.entriesByConversation} ON ${tables.entries} (conversation_id, id DESC)`,
+		`CREATE INDEX ${indexes.entryHeadsByConversation} ON ${tables.entries} (conversation_id, id DESC) WHERE head IS NOT NULL`,
+		`CREATE TABLE ${tables.tasks} (
+			id INTEGER PRIMARY KEY,
+			conversation_id INTEGER NOT NULL,
+			kind TEXT NOT NULL,
+			status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'waiting', 'completing', 'terminal')),
+			abort_requested INTEGER NOT NULL CHECK (abort_requested IN (0, 1)),
+			background INTEGER NOT NULL CHECK (background IN (0, 1)),
+			record TEXT NOT NULL CHECK (json_valid(record))
+		) STRICT`,
+		`CREATE INDEX ${indexes.tasksByStatus} ON ${tables.tasks} (status, id)`,
+		`CREATE INDEX ${indexes.tasksByConversation} ON ${tables.tasks} (conversation_id, id)`,
+		`CREATE INDEX ${indexes.tasksByKind} ON ${tables.tasks} (kind, id)`,
+		`CREATE INDEX ${indexes.tasksByAbortRequested} ON ${tables.tasks} (abort_requested, id)`,
+		`CREATE INDEX ${indexes.tasksByBackground} ON ${tables.tasks} (background, id)`,
+		`CREATE TABLE ${tables.submissions} (
+			id INTEGER PRIMARY KEY,
+			conversation_id INTEGER NOT NULL,
+			request_id TEXT,
+			status TEXT NOT NULL CHECK (status IN ('queued', 'placed', 'done', 'unanswered')),
+			record TEXT NOT NULL CHECK (json_valid(record))
+		) STRICT`,
+		`CREATE INDEX ${indexes.submissionsByRequest} ON ${tables.submissions} (conversation_id, request_id)`,
+		`CREATE INDEX ${indexes.submissionsByConversation} ON ${tables.submissions} (conversation_id, id)`,
+		`CREATE INDEX ${indexes.submissionsByStatus} ON ${tables.submissions} (status, id)`,
+		`CREATE TABLE ${tables.documents} (
+			id INTEGER PRIMARY KEY,
+			kind TEXT NOT NULL,
+			family INTEGER NOT NULL CHECK (family IN (0, 1)),
+			key_value TEXT NOT NULL,
+			scope_kind TEXT NOT NULL CHECK (scope_kind IN ('session', 'conversation', 'task')),
+			owner_id INTEGER NOT NULL,
+			created_at INTEGER NOT NULL,
+			retired_at INTEGER,
+			record TEXT NOT NULL CHECK (json_valid(record))
+		) STRICT`,
+		`CREATE INDEX ${indexes.documentsByAddress}
+			ON ${tables.documents} (kind, scope_kind, owner_id, family, key_value, created_at DESC, retired_at)`,
+		`CREATE INDEX ${indexes.documentsByScope} ON ${tables.documents} (scope_kind, owner_id, id)`,
+		`CREATE INDEX ${indexes.documentsByScopeKind} ON ${tables.documents} (scope_kind, owner_id, kind, id)`,
+		`CREATE TABLE ${tables.documentRevisions} (
+			document_id INTEGER NOT NULL,
+			seq INTEGER NOT NULL,
+			kind TEXT NOT NULL CHECK (kind IN ('base', 'delta')),
+			version INTEGER NOT NULL,
+			content TEXT NOT NULL CHECK (json_valid(content)),
+			PRIMARY KEY (document_id, seq)
+		) STRICT`,
+		`CREATE INDEX ${indexes.documentRevisionsByKind} ON ${tables.documentRevisions} (document_id, kind, seq DESC)`,
+	];
+	return [{ version: 1, statements: initialSchema }];
+}
+
+export const SQLITE_MIGRATIONS: readonly SqliteMigration[] = createSqliteMigrations(DEFAULT_SQLITE_SCHEMA);
+
+export const CURRENT_SQLITE_SCHEMA_VERSION = SQLITE_MIGRATIONS.at(-1)?.version ?? 0;
+
+type SchemaRow = { readonly version: number };
+
+/** Apply all pending schema migrations atomically. */
+export async function applySqliteMigrations(
+	database: SqliteDatabase,
+	migrations: readonly SqliteMigration[] = SQLITE_MIGRATIONS,
+): Promise<void> {
+	await applySqliteMigrationsForSchema(database, DEFAULT_SQLITE_SCHEMA, migrations);
+}
+
+/** Apply migrations rendered for a particular schema. */
+export async function applySqliteMigrationsForSchema(
+	database: SqliteDatabase,
+	schema: SqliteSchema,
+	migrations: readonly SqliteMigration[] = createSqliteMigrations(schema),
+): Promise<void> {
+	for (let index = 0; index < migrations.length; index++) {
+		if (migrations[index]?.version !== index + 1) {
+			throw new Error("Durable SQLite migrations must have contiguous versions starting at 1");
+		}
+	}
+
+	const { durableSchema } = schema.tables;
+	await database.transaction(async (transaction) => {
+		await transaction.exec(`CREATE TABLE IF NOT EXISTS ${durableSchema} (
+			singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
+			version INTEGER NOT NULL CHECK (version >= 0)
+		) STRICT`);
+		await transaction.run(`INSERT OR IGNORE INTO ${durableSchema} (singleton, version) VALUES (1, 0)`);
+		const row = await transaction.get<SchemaRow>(`SELECT version FROM ${durableSchema} WHERE singleton = 1`);
+		if (row === undefined) throw new Error("Durable SQLite schema metadata is missing");
+		const currentVersion = migrations.at(-1)?.version ?? 0;
+		if (row.version > currentVersion) {
+			throw new Error(
+				`Durable SQLite schema version ${row.version} is newer than supported version ${currentVersion}`,
+			);
+		}
+		for (const migration of migrations) {
+			if (migration.version <= row.version) continue;
+			for (const statement of migration.statements) await transaction.exec(statement);
+			await transaction.run(`UPDATE ${durableSchema} SET version = ? WHERE singleton = 1`, migration.version);
+		}
+	});
+}

third_party/pi/packages/durable/src/storage/sqlite/node.ts

@@ -0,0 +1,216 @@
+import { mkdir } from "node:fs/promises";
+import { dirname } from "node:path";
+import type { StatementSync } from "node:sqlite";
+import { DatabaseSync } from "node:sqlite";
+import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "./database.ts";
+import { validateSqliteSchemaPrefix } from "./schema.ts";
+import { SqliteStorage, type SqliteStorageOptions } from "./storage.ts";
+
+/** Node SQLite connection settings for a durable storage file. */
+export type NodeSqliteDatabaseOptions = {
+	/** SQLite WAL auto-checkpoint threshold. SQLite and this adapter default to 1,000 pages; 0 disables it. */
+	readonly walAutoCheckpointPages?: number;
+	/** Time SQLite waits for a competing file lock. SQLite defaults to 0; this adapter defaults to 5,000 ms. */
+	readonly busyTimeoutMs?: number;
+};
+
+/** Node SQLite connection settings plus durable storage options. */
+export type NodeSqliteStorageOptions = NodeSqliteDatabaseOptions & SqliteStorageOptions;
+
+const DEFAULT_WAL_AUTO_CHECKPOINT_PAGES = 1_000;
+const DEFAULT_BUSY_TIMEOUT_MS = 5_000;
+
+type TransactionScope = { active: boolean };
+
+const ignore = (): void => {};
+
+/**
+ * Runs operations in call order. An operation starts immediately when nothing is running or waiting;
+ * otherwise it waits for everything before it. An asynchronous operation holds the queue until it settles.
+ */
+class SerialOperationQueue {
+	private tail: Promise<void> = Promise.resolve();
+	private pending = 0;
+
+	run<T>(operation: () => T): Promise<T> {
+		if (this.pending > 0) return this.enqueue(operation);
+		try {
+			return Promise.resolve(operation());
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+	runAsync<T>(operation: () => Promise<T>): Promise<T> {
+		if (this.pending > 0) return this.enqueue(operation);
+		this.pending++;
+		// Publish the barrier before the operation starts, so calls it makes synchronously wait behind it.
+		const { promise: barrier, resolve: releaseBarrier } = Promise.withResolvers<void>();
+		this.tail = barrier;
+		let started: Promise<T>;
+		try {
+			started = operation();
+		} catch (error) {
+			started = Promise.reject(error);
+		}
+		return started.finally(() => {
+			this.pending--;
+			releaseBarrier();
+		});
+	}
+
+	private enqueue<T>(operation: () => T | Promise<T>): Promise<T> {
+		this.pending++;
+		return this.release(this.tail.then(operation));
+	}
+
+	private release<T>(operation: Promise<T>): Promise<T> {
+		const settled = operation.finally(() => {
+			this.pending--;
+		});
+		this.tail = settled.then(ignore, ignore);
+		return settled;
+	}
+}
+
+/**
+ * Executes SQL on one connection. Prepared statements are cached per connection by SQL text, so the
+ * database and its transaction handles share them across transactions.
+ */
+abstract class NodeSqliteExecutor implements SqliteExecutor {
+	protected readonly database: DatabaseSync;
+	protected readonly statements: Map<string, StatementSync>;
+
+	constructor(database: DatabaseSync, statements: Map<string, StatementSync>) {
+		this.database = database;
+		this.statements = statements;
+	}
+
+	exec(sql: string): Promise<void> {
+		return this.runOperation(() => {
+			this.database.exec(sql);
+		});
+	}
+
+	run(sql: string, ...params: SqliteValue[]): Promise<void> {
+		return this.runOperation(() => {
+			this.statement(sql).run(...params);
+		});
+	}
+
+	get<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T | undefined> {
+		return this.runOperation(() => this.statement(sql).get(...params) as T | undefined);
+	}
+
+	all<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T[]> {
+		return this.runOperation(() => this.statement(sql).all(...params) as T[]);
+	}
+
+	protected abstract runOperation<T>(operation: () => T): Promise<T>;
+
+	private statement(sql: string): StatementSync {
+		let statement = this.statements.get(sql);
+		if (statement === undefined) {
+			statement = this.database.prepare(sql);
+			this.statements.set(sql, statement);
+		}
+		return statement;
+	}
+}
+
+class NodeSqliteTransaction extends NodeSqliteExecutor {
+	private readonly scope: TransactionScope;
+
+	constructor(database: DatabaseSync, statements: Map<string, StatementSync>, scope: TransactionScope) {
+		super(database, statements);
+		this.scope = scope;
+	}
+
+	protected async runOperation<T>(operation: () => T): Promise<T> {
+		if (!this.scope.active) throw new Error("SQLite transaction handle is no longer active");
+		return operation();
+	}
+}
+
+/** `SqliteDatabase` adapter backed by Node's built-in `node:sqlite`. */
+export class NodeSqliteDatabase extends NodeSqliteExecutor implements SqliteDatabase {
+	private readonly access = new SerialOperationQueue();
+	private closed = false;
+
+	constructor(database: DatabaseSync) {
+		super(database, new Map());
+	}
+
+	transaction<T>(callback: (transaction: SqliteExecutor) => Promise<T>): Promise<T> {
+		return this.access.runAsync(async () => {
+			this.database.exec("BEGIN IMMEDIATE");
+			const scope = { active: true };
+			try {
+				const result = await callback(new NodeSqliteTransaction(this.database, this.statements, scope));
+				scope.active = false;
+				this.database.exec("COMMIT");
+				return result;
+			} catch (error) {
+				scope.active = false;
+				try {
+					this.database.exec("ROLLBACK");
+				} catch (rollbackError) {
+					throw new AggregateError([error, rollbackError], "SQLite transaction failed and rollback failed");
+				}
+				throw error;
+			}
+		});
+	}
+
+	close(): Promise<void> {
+		return this.access.run(() => {
+			if (this.closed) return;
+			this.closed = true;
+			this.statements.clear();
+			try {
+				this.database.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+			} finally {
+				this.database.close();
+			}
+		});
+	}
+
+	protected runOperation<T>(operation: () => T): Promise<T> {
+		return this.access.run(operation);
+	}
+}
+
+/** Open and configure a Node-backed SQLite database facade. */
+export async function openNodeSqliteDatabase(
+	path: string,
+	options: NodeSqliteDatabaseOptions = {},
+): Promise<NodeSqliteDatabase> {
+	const checkpointPages = options.walAutoCheckpointPages ?? DEFAULT_WAL_AUTO_CHECKPOINT_PAGES;
+	const timeout = options.busyTimeoutMs ?? DEFAULT_BUSY_TIMEOUT_MS;
+	if (path !== ":memory:") await mkdir(dirname(path), { recursive: true });
+	const database = new DatabaseSync(path, { timeout });
+	const adapter = new NodeSqliteDatabase(database);
+	try {
+		await adapter.exec("PRAGMA journal_mode = WAL");
+		await adapter.exec("PRAGMA synchronous = NORMAL");
+		await adapter.exec(`PRAGMA wal_autocheckpoint = ${checkpointPages}`);
+		return adapter;
+	} catch (error) {
+		try {
+			await adapter.close();
+		} catch {
+			// Preserve the configuration failure.
+		}
+		throw error;
+	}
+}
+
+/** Open or create file-backed durable storage using Node's built-in SQLite, optionally under a table prefix. */
+export async function openNodeSqliteStorage(
+	path: string,
+	options: NodeSqliteStorageOptions = {},
+): Promise<SqliteStorage> {
+	const { prefix, ...databaseOptions } = options;
+	if (prefix !== undefined) validateSqliteSchemaPrefix(prefix);
+	return SqliteStorage.open(await openNodeSqliteDatabase(path, databaseOptions), { prefix });
+}

third_party/pi/packages/durable/src/storage/sqlite/schema.ts

@@ -0,0 +1,69 @@
+/**
+ * Immutable identifier history for migrations and runtime queries. Never change or remove an existing value; a
+ * rename adds a new entry so historical migrations keep rendering their original object names.
+ */
+const TABLE_NAMES = {
+	durableSchema: "durable_schema",
+	durableMetadata: "durable_metadata",
+	recordIds: "record_ids",
+	conversations: "conversations",
+	entries: "entries",
+	tasks: "tasks",
+	submissions: "submissions",
+	documents: "documents",
+	documentRevisions: "document_revisions",
+} as const;
+
+const INDEX_NAMES = {
+	conversationsByOwnerConversation: "conversations_by_owner_conversation",
+	conversationsByOwnerTask: "conversations_by_owner_task",
+	entriesByConversation: "entries_by_conversation",
+	entryHeadsByConversation: "entry_heads_by_conversation",
+	tasksByStatus: "tasks_by_status",
+	tasksByConversation: "tasks_by_conversation",
+	tasksByKind: "tasks_by_kind",
+	tasksByAbortRequested: "tasks_by_abort_requested",
+	tasksByBackground: "tasks_by_background",
+	submissionsByRequest: "submissions_by_request",
+	submissionsByConversation: "submissions_by_conversation",
+	submissionsByStatus: "submissions_by_status",
+	documentsByAddress: "documents_by_address",
+	documentsByScope: "documents_by_scope",
+	documentsByScopeKind: "documents_by_scope_kind",
+	documentRevisionsByKind: "document_revisions_by_kind",
+} as const;
+
+type SqliteIdentifiers<T extends Record<string, string>> = { readonly [K in keyof T]: string };
+
+export type SqliteSchema = {
+	readonly tables: SqliteIdentifiers<typeof TABLE_NAMES>;
+	readonly indexes: SqliteIdentifiers<typeof INDEX_NAMES>;
+};
+
+/** Validate a prefix before using it as part of a SQLite identifier. */
+export function validateSqliteSchemaPrefix(prefix: string): void {
+	if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(prefix)) {
+		throw new Error(`Invalid SQLite schema prefix ${JSON.stringify(prefix)}`);
+	}
+	if (prefix.toLowerCase().startsWith("sqlite_")) {
+		throw new Error(`SQLite schema prefix ${JSON.stringify(prefix)} is reserved`);
+	}
+}
+
+function prefixIdentifiers<T extends Record<string, string>>(names: T, prefix: string): SqliteIdentifiers<T> {
+	return Object.fromEntries(
+		Object.entries(names).map(([key, name]) => [key, `${prefix}${name}`]),
+	) as SqliteIdentifiers<T>;
+}
+
+/** Build the identifiers used by every migration and storage query. */
+export function createSqliteSchema(prefix?: string): SqliteSchema {
+	if (prefix !== undefined) validateSqliteSchemaPrefix(prefix);
+	const actualPrefix = prefix ?? "";
+	return {
+		tables: prefixIdentifiers(TABLE_NAMES, actualPrefix),
+		indexes: prefixIdentifiers(INDEX_NAMES, actualPrefix),
+	};
+}
+
+export const DEFAULT_SQLITE_SCHEMA = createSqliteSchema();

third_party/pi/packages/durable/src/storage/sqlite/storage.ts

@@ -0,0 +1,957 @@
+import type { Context, JsonValue } from "@earendil-works/chord";
+import { apply, type Op } from "@earendil-works/chord/delta";
+import { StorageRequestError } from "../../errors.ts";
+import { idFromNumber, seqFromNumber } from "../../ids.ts";
+import type {
+	ConversationId,
+	ConversationQuery,
+	ConversationRecord,
+	Cursor,
+	DocumentAddress,
+	DocumentContent,
+	DocumentCreate,
+	DocumentId,
+	DocumentPoint,
+	DocumentQuery,
+	DocumentRecord,
+	EntryId,
+	EntryQuery,
+	EntryRecord,
+	Id,
+	JsonObject,
+	Page,
+	ScanOrder,
+	Seq,
+	Storage,
+	StorageWrite,
+	StoredDocument,
+	SubmissionId,
+	SubmissionQuery,
+	SubmissionRecord,
+	TaskId,
+	TaskQuery,
+	TaskRecord,
+} from "../../types.ts";
+import { nextCursor, type ScanStart, scanStart } from "../scan.ts";
+import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "./database.ts";
+import { applySqliteMigrationsForSchema } from "./migrations.ts";
+import { createSqliteSchema, type SqliteSchema } from "./schema.ts";
+
+type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
+type TableName = "conversation" | "entry" | "task" | "submission" | "document";
+type RecordIdRow = { readonly record_type: TableName };
+type JsonRow = { readonly record: string };
+type EntryJsonRow = { readonly record: string; readonly commit_seq: number };
+type RevisionRow = {
+	readonly seq: number;
+	readonly kind: DocumentContent["kind"];
+	readonly version: number;
+	readonly content: string;
+};
+type IdRow = { readonly id: number };
+type MetadataRow = { readonly next_id: string; readonly next_seq: number };
+type DocumentAction = {
+	create?: DocumentCreate;
+	copy?: Extract<StorageWrite, { readonly type: "document.copy" }>["source"];
+	content?: DocumentContent;
+	retire: boolean;
+};
+type ScopeColumns = {
+	readonly scopeKind: DocumentRecord["scope"]["kind"];
+	readonly ownerId: number;
+};
+
+export type SqliteStorageOptions = {
+	/**
+	 * Prefix for every table and index in the durable schema. It must be a SQLite identifier and must not use
+	 * SQLite's reserved `sqlite_` prefix. Existing data is not migrated when this changes.
+	 */
+	readonly prefix?: string;
+};
+
+const parseJson = <T>(value: string): T => JSON.parse(value) as T;
+const encodeJson = (value: unknown): string => JSON.stringify(value) as string;
+// Some SQLite bindings replace lone UTF-16 surrogates. JSON encoding keeps indexed identities lossless.
+const encodeIndexedString = (value: string): string => JSON.stringify(value);
+
+const cursorId = <I extends Id<string>>(cursor: Cursor | undefined): I | undefined => {
+	const after = cursor?.after;
+	if (after === undefined) return undefined;
+	if (typeof after !== "number" || !Number.isSafeInteger(after))
+		throw new StorageRequestError("Invalid storage cursor");
+	return idFromNumber<I>(after);
+};
+
+const page = <T extends { readonly id: Id<string> }>(
+	values: readonly T[],
+	limit: number,
+	order: ScanOrder = "ascending",
+): Page<T, Cursor> => {
+	const items = values.slice(0, limit);
+	if (values.length <= limit) return { items };
+	return { items, next: nextCursor(items.at(-1)!.id, order) };
+};
+
+/** The ID condition, its parameter, and the `ORDER BY` direction of a table scan. */
+const scanSql = (start: ScanStart): { readonly clause: string; readonly param: number; readonly direction: string } =>
+	start.order === "ascending"
+		? { clause: "id > ?", param: start.after ?? -1, direction: "ASC" }
+		: { clause: "id < ?", param: start.after ?? Number.MAX_SAFE_INTEGER, direction: "DESC" };
+
+const scopeColumns = (scope: DocumentRecord["scope"]): ScopeColumns => {
+	switch (scope.kind) {
+		case "session":
+			return { scopeKind: "session", ownerId: 0 };
+		case "conversation":
+			return { scopeKind: "conversation", ownerId: scope.conversationId };
+		case "task":
+			return { scopeKind: "task", ownerId: scope.taskId };
+	}
+};
+
+const addressParts = (address: DocumentAddress | DocumentCreate | DocumentRecord) => {
+	const scope = scopeColumns(address.scope);
+	return {
+		kind: encodeIndexedString(address.kind),
+		...scope,
+		family: address.key === undefined ? 0 : 1,
+		keyValue: encodeIndexedString(address.key ?? ""),
+	};
+};
+
+const addressKey = (address: DocumentAddress | DocumentCreate | DocumentRecord): string => {
+	const parts = addressParts(address);
+	return JSON.stringify([parts.kind, parts.scopeKind, parts.ownerId, parts.family, parts.keyValue]);
+};
+
+const isAliveAt = (record: DocumentRecord, at: DocumentPoint): boolean => {
+	if (at === "current") return record.retiredAt === undefined;
+	return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt);
+};
+
+const isCurrentOnly = (record: DocumentRecord): boolean =>
+	record.scope.kind !== "conversation" || record.history === "latest";
+
+const writeId = (write: StorageWrite): Id<string> | undefined => {
+	switch (write.type) {
+		case "conversation":
+		case "entry":
+		case "task":
+		case "submission":
+			return write.value.id;
+		case "document.create":
+		case "document.copy":
+			return write.record.id;
+		case "document.change":
+		case "document.retire":
+			return undefined;
+	}
+};
+
+/** Portable SQLite implementation of the Pico storage contract. */
+export class SqliteStorage implements Storage {
+	private readonly db: SqliteDatabase;
+	private readonly schema: SqliteSchema;
+	private nextId: number;
+	private closed = false;
+	private closing: Promise<void> | undefined;
+	private admittedReads = 0;
+	private readsDrained: (() => void) | undefined;
+
+	private constructor(db: SqliteDatabase, schema: SqliteSchema, nextId: number) {
+		this.db = db;
+		this.schema = schema;
+		this.nextId = nextId;
+	}
+
+	/** Initialize storage over an owned SQLite database facade. */
+	static async open(db: SqliteDatabase, options: SqliteStorageOptions = {}): Promise<SqliteStorage> {
+		try {
+			const schema = createSqliteSchema(options.prefix);
+			await applySqliteMigrationsForSchema(db, schema);
+			const metadata = await db.get<MetadataRow>(
+				`SELECT next_id, next_seq FROM ${schema.tables.durableMetadata} WHERE singleton = 1`,
+			);
+			if (metadata === undefined) throw new Error("Durable SQLite metadata is missing");
+			return new SqliteStorage(db, schema, Number(metadata.next_id));
+		} catch (error) {
+			try {
+				await db.close();
+			} catch {
+				// Preserve the initialization failure.
+			}
+			throw error;
+		}
+	}
+
+	async commit(writes: readonly StorageWrite[], _context: Context): Promise<Seq> {
+		this.assertOpen();
+		const documentActions = this.prepareDocumentActions(writes);
+		const candidateNextId = this.candidateNextId(writes);
+		const seq = await this.db.transaction(async (transaction) => {
+			const metadata = await transaction.get<MetadataRow>(
+				`SELECT next_id, next_seq FROM ${this.schema.tables.durableMetadata} WHERE singleton = 1`,
+			);
+			if (metadata === undefined) throw new Error("Durable SQLite metadata is missing");
+			const committedSeq = seqFromNumber(metadata.next_seq);
+			await this.checkGlobalIds(transaction, writes);
+			await this.checkDocumentActions(transaction, documentActions);
+			for (const write of writes) await this.applyTableWrite(transaction, write, committedSeq);
+			await this.applyDocumentActions(transaction, documentActions, committedSeq);
+			await transaction.run(
+				`UPDATE ${this.schema.tables.durableMetadata} SET next_id = ?, next_seq = ? WHERE singleton = 1`,
+				String(Math.max(Number(metadata.next_id), candidateNextId)),
+				committedSeq + 1,
+			);
+			return committedSeq;
+		});
+		this.nextId = Math.max(this.nextId, candidateNextId);
+		return seq;
+	}
+
+	async mintId<I extends Id<string>>(): Promise<I> {
+		this.assertOpen();
+		if (!Number.isSafeInteger(this.nextId)) throw new Error("ID space is exhausted");
+		return idFromNumber<I>(this.nextId++);
+	}
+
+	async conversation(id: ConversationId, _context: Context): Promise<ConversationRecord | undefined> {
+		this.assertOpen();
+		const row = await this.db.get<JsonRow>(`SELECT record FROM ${this.schema.tables.conversations} WHERE id = ?`, id);
+		return row === undefined ? undefined : parseJson<ConversationRecord>(row.record);
+	}
+
+	async scanConversations(
+		query: ConversationQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<ConversationRecord, Cursor>> {
+		this.assertOpen();
+		const start = scanStart(query.order, cursor, "ascending");
+		const scan = scanSql(start);
+		const clauses = [scan.clause];
+		const params: SqliteValue[] = [scan.param];
+		if (query.ownerConversationId !== undefined) {
+			clauses.push("owner_conversation_id = ?");
+			params.push(query.ownerConversationId);
+		}
+		if (query.ownerTaskId !== undefined) {
+			clauses.push("owner_task_id = ?");
+			params.push(query.ownerTaskId);
+		}
+		params.push(limit + 1);
+		const rows = await this.db.all<JsonRow>(
+			`SELECT record FROM ${this.schema.tables.conversations} WHERE ${clauses.join(" AND ")} ORDER BY id ${scan.direction} LIMIT ?`,
+			...params,
+		);
+		return page(
+			rows.map((row) => parseJson<ConversationRecord>(row.record)),
+			limit,
+			start.order,
+		);
+	}
+
+	entry(id: EntryId, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
+	entry(
+		conversationId: ConversationId,
+		id: EntryId,
+		context: Context,
+	): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
+	entry(
+		idOrConversationId: EntryId | ConversationId,
+		idOrContext: EntryId | Context,
+		context?: Context,
+	): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined> {
+		return this.admitRead(() => this.readEntry(idOrConversationId, idOrContext, context));
+	}
+
+	findLatestHeadMarker(
+		conversationId: ConversationId,
+		atOrBeforeEntryId: EntryId | undefined,
+		_context: Context,
+	): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> {
+		return this.admitRead(() => this.readLatestHeadMarker(conversationId, atOrBeforeEntryId));
+	}
+
+	scanEntries(
+		query: EntryQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<EntryRecord, Cursor>> {
+		return this.admitRead(() => this.readEntries(query, limit, cursor));
+	}
+
+	private async readEntry(
+		idOrConversationId: EntryId | ConversationId,
+		idOrContext: EntryId | Context,
+		context?: Context,
+	): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined> {
+		const id =
+			context === undefined
+				? idFromNumber<EntryId>(idOrConversationId)
+				: typeof idOrContext === "number"
+					? idFromNumber<EntryId>(idOrContext)
+					: undefined;
+		if (id === undefined) throw new StorageRequestError("Storage.entry() requires an entry ID");
+		let conversation: ConversationRecord | undefined;
+		if (context !== undefined) {
+			const conversationId = idFromNumber<ConversationId>(idOrConversationId);
+			conversation = await this.readConversation(conversationId);
+			if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${conversationId}`);
+		}
+		const row = await this.db.get<EntryJsonRow>(
+			`SELECT record, commit_seq FROM ${this.schema.tables.entries} WHERE id = ?`,
+			id,
+		);
+		if (row === undefined) return undefined;
+		const entry = parseJson<EntryRecord>(row.record);
+		if (conversation !== undefined) {
+			let upperEntryId = Number.POSITIVE_INFINITY;
+			while (conversation.id !== entry.conversationId) {
+				if (conversation.parent === undefined) return undefined;
+				upperEntryId = Math.min(upperEntryId, conversation.parent.at);
+				conversation = (await this.readConversation(conversation.parent.conversationId))!;
+			}
+			if (entry.id > upperEntryId) return undefined;
+		}
+		return { entry, commitSeq: seqFromNumber(row.commit_seq) };
+	}
+
+	private async readLatestHeadMarker(
+		conversationId: ConversationId,
+		atOrBeforeEntryId: EntryId | undefined,
+	): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> {
+		let conversation = await this.readConversation(conversationId);
+		if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${conversationId}`);
+		let upper: number | undefined = atOrBeforeEntryId;
+		while (true) {
+			const row =
+				upper === undefined
+					? await this.db.get<JsonRow>(
+							`SELECT record FROM ${this.schema.tables.entries} WHERE conversation_id = ? AND head IS NOT NULL ORDER BY id DESC LIMIT 1`,
+							conversation.id,
+						)
+					: await this.db.get<JsonRow>(
+							`SELECT record FROM ${this.schema.tables.entries} WHERE conversation_id = ? AND head IS NOT NULL AND id <= ? ORDER BY id DESC LIMIT 1`,
+							conversation.id,
+							upper,
+						);
+			if (row !== undefined) return parseJson<EntryRecord & { readonly head: EntryId }>(row.record);
+			if (conversation.parent === undefined) return undefined;
+			upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at);
+			conversation = (await this.readConversation(conversation.parent.conversationId))!;
+		}
+	}
+
+	private async readEntries(
+		query: EntryQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+	): Promise<Page<EntryRecord, Cursor>> {
+		const { order, after } = scanStart(query.order, cursor, "descending");
+		if (order === "ascending") return this.readEntriesAscending(query, limit, after);
+		let conversation = await this.readConversation(query.conversationId);
+		if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${query.conversationId}`);
+		let upper: number | undefined = query.maxEntryId;
+		if (after !== undefined) upper = Math.min(upper ?? Number.MAX_SAFE_INTEGER, after - 1);
+		const values: EntryRecord[] = [];
+		while (true) {
+			const clauses = ["conversation_id = ?"];
+			const params: SqliteValue[] = [conversation.id];
+			if (query.minEntryId !== undefined) {
+				clauses.push("id >= ?");
+				params.push(query.minEntryId);
+			}
+			if (upper !== undefined) {
+				clauses.push("id <= ?");
+				params.push(upper);
+			}
+			params.push(limit + 1 - values.length);
+			const rows = await this.db.all<JsonRow>(
+				`SELECT record FROM ${this.schema.tables.entries} WHERE ${clauses.join(" AND ")} ORDER BY id DESC LIMIT ?`,
+				...params,
+			);
+			values.push(...rows.map((row) => parseJson<EntryRecord>(row.record)));
+			if (values.length > limit || conversation.parent === undefined) break;
+			upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at);
+			if (query.minEntryId !== undefined && upper < query.minEntryId) break;
+			conversation = (await this.readConversation(conversation.parent.conversationId))!;
+		}
+		return page(values, limit, "descending");
+	}
+
+	/** Oldest first: the fork chain's segments from the root conversation forward, each up to its fork point. */
+	private async readEntriesAscending(
+		query: EntryQuery,
+		limit: number,
+		after: number | undefined,
+	): Promise<Page<EntryRecord, Cursor>> {
+		const segments: { readonly conversationId: ConversationId; readonly upper: number | undefined }[] = [];
+		let conversation = await this.readConversation(query.conversationId);
+		if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${query.conversationId}`);
+		let upper: number | undefined = query.maxEntryId;
+		while (true) {
+			segments.push({ conversationId: conversation.id, upper });
+			if (conversation.parent === undefined) break;
+			upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at);
+			if (query.minEntryId !== undefined && upper < query.minEntryId) break;
+			conversation = (await this.readConversation(conversation.parent.conversationId))!;
+		}
+		let lower: number | undefined = query.minEntryId;
+		if (after !== undefined) lower = Math.max(lower ?? after + 1, after + 1);
+		const values: EntryRecord[] = [];
+		for (const segment of segments.reverse()) {
+			const clauses = ["conversation_id = ?"];
+			const params: SqliteValue[] = [segment.conversationId];
+			if (lower !== undefined) {
+				clauses.push("id >= ?");
+				params.push(lower);
+			}
+			if (segment.upper !== undefined) {
+				clauses.push("id <= ?");
+				params.push(segment.upper);
+			}
+			params.push(limit + 1 - values.length);
+			const rows = await this.db.all<JsonRow>(
+				`SELECT record FROM ${this.schema.tables.entries} WHERE ${clauses.join(" AND ")} ORDER BY id ASC LIMIT ?`,
+				...params,
+			);
+			values.push(...rows.map((row) => parseJson<EntryRecord>(row.record)));
+			if (values.length > limit) break;
+		}
+		return page(values, limit, "ascending");
+	}
+
+	async task(id: TaskId, _context: Context): Promise<StoredTask | undefined> {
+		this.assertOpen();
+		const row = await this.db.get<JsonRow>(`SELECT record FROM ${this.schema.tables.tasks} WHERE id = ?`, id);
+		return row === undefined ? undefined : parseJson<StoredTask>(row.record);
+	}
+
+	async scanTasks(
+		query: TaskQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<StoredTask, Cursor>> {
+		this.assertOpen();
+		const start = scanStart(query.order, cursor, "ascending");
+		const scan = scanSql(start);
+		const clauses = [scan.clause];
+		const params: SqliteValue[] = [scan.param];
+		if (query.conversationId !== undefined) {
+			clauses.push("conversation_id = ?");
+			params.push(query.conversationId);
+		}
+		if (query.kind !== undefined) {
+			clauses.push("kind = ?");
+			params.push(encodeIndexedString(query.kind));
+		}
+		if (query.status !== undefined) {
+			clauses.push("status = ?");
+			params.push(query.status);
+		}
+		if (query.abortRequested !== undefined) {
+			clauses.push("abort_requested = ?");
+			params.push(query.abortRequested ? 1 : 0);
+		}
+		if (query.background !== undefined) {
+			clauses.push("background = ?");
+			params.push(query.background ? 1 : 0);
+		}
+		params.push(limit + 1);
+		const rows = await this.db.all<JsonRow>(
+			`SELECT record FROM ${this.schema.tables.tasks} WHERE ${clauses.join(" AND ")} ORDER BY id ${scan.direction} LIMIT ?`,
+			...params,
+		);
+		return page(
+			rows.map((row) => parseJson<StoredTask>(row.record)),
+			limit,
+			start.order,
+		);
+	}
+
+	async submission(id: SubmissionId, _context: Context): Promise<SubmissionRecord | undefined> {
+		this.assertOpen();
+		const row = await this.db.get<JsonRow>(`SELECT record FROM ${this.schema.tables.submissions} WHERE id = ?`, id);
+		return row === undefined ? undefined : parseJson<SubmissionRecord>(row.record);
+	}
+
+	async scanSubmissions(
+		query: SubmissionQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<SubmissionRecord, Cursor>> {
+		this.assertOpen();
+		const start = scanStart(query.order, cursor, "ascending");
+		const scan = scanSql(start);
+		const clauses = [scan.clause];
+		const params: SqliteValue[] = [scan.param];
+		if (query.conversationId !== undefined) {
+			clauses.push("conversation_id = ?");
+			params.push(query.conversationId);
+		}
+		if (query.status !== undefined) {
+			clauses.push("status = ?");
+			params.push(query.status);
+		}
+		params.push(limit + 1);
+		const rows = await this.db.all<JsonRow>(
+			`SELECT record FROM ${this.schema.tables.submissions} WHERE ${clauses.join(" AND ")} ORDER BY id ${scan.direction} LIMIT ?`,
+			...params,
+		);
+		return page(
+			rows.map((row) => parseJson<SubmissionRecord>(row.record)),
+			limit,
+			start.order,
+		);
+	}
+
+	async submissionByRequest(
+		conversationId: ConversationId,
+		requestId: string,
+		_context: Context,
+	): Promise<SubmissionRecord | undefined> {
+		this.assertOpen();
+		const row = await this.db.get<JsonRow>(
+			`SELECT record FROM ${this.schema.tables.submissions} WHERE conversation_id = ? AND request_id = ?`,
+			conversationId,
+			encodeIndexedString(requestId),
+		);
+		return row === undefined ? undefined : parseJson<SubmissionRecord>(row.record);
+	}
+
+	async findDocument(
+		address: DocumentAddress,
+		at: DocumentPoint,
+		_context: Context,
+	): Promise<DocumentRecord | undefined> {
+		this.assertOpen();
+		const parts = addressParts(address);
+		const sql =
+			at === "current"
+				? `SELECT record FROM ${this.schema.tables.documents}
+					WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ?
+					AND retired_at IS NULL ORDER BY created_at DESC LIMIT 1`
+				: `SELECT record FROM ${this.schema.tables.documents}
+					WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ?
+					AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?)
+					ORDER BY created_at DESC LIMIT 1`;
+		const params: SqliteValue[] = [parts.kind, parts.scopeKind, parts.ownerId, parts.family, parts.keyValue];
+		if (at !== "current") params.push(at, at);
+		const row = await this.db.get<JsonRow>(sql, ...params);
+		return row === undefined ? undefined : parseJson<DocumentRecord>(row.record);
+	}
+
+	async document(id: DocumentId, at: DocumentPoint, _context: Context): Promise<StoredDocument | undefined> {
+		this.assertOpen();
+		// The record and revision queries must observe one committed state; a commit between them can replace the base.
+		return this.db.transaction((transaction) => this.materializeDocument(transaction, id, at));
+	}
+
+	async scanDocuments(
+		query: DocumentQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<DocumentRecord, Cursor>> {
+		this.assertOpen();
+		const scope = scopeColumns(query.scope);
+		const clauses = ["scope_kind = ?", "owner_id = ?", "id > ?"];
+		const params: SqliteValue[] = [scope.scopeKind, scope.ownerId, cursorId(cursor) ?? -1];
+		if (query.kind !== undefined) {
+			clauses.push("kind = ?");
+			params.push(encodeIndexedString(query.kind));
+		}
+		if (query.at === "current") {
+			clauses.push("retired_at IS NULL");
+		} else {
+			clauses.push("created_at <= ?", "(retired_at IS NULL OR retired_at > ?)");
+			params.push(query.at, query.at);
+		}
+		params.push(limit + 1);
+		const rows = await this.db.all<JsonRow>(
+			`SELECT record FROM ${this.schema.tables.documents} WHERE ${clauses.join(" AND ")} ORDER BY id LIMIT ?`,
+			...params,
+		);
+		return page(
+			rows.map((row) => parseJson<DocumentRecord>(row.record)),
+			limit,
+		);
+	}
+
+	close(_context: Context): Promise<void> {
+		if (this.closing === undefined) {
+			this.closed = true;
+			this.closing = this.closeDatabase();
+		}
+		return this.closing;
+	}
+
+	private async closeDatabase(): Promise<void> {
+		if (this.admittedReads > 0) {
+			await new Promise<void>((resolve) => {
+				this.readsDrained = resolve;
+			});
+		}
+		await this.db.close();
+	}
+
+	/**
+	 * Run a read that issues several queries. Close waits for admitted reads, so their later queries never reach a
+	 * closed database. Single-query reads and transactions are already ordered before close by the database.
+	 */
+	private async admitRead<T>(read: () => Promise<T>): Promise<T> {
+		this.assertOpen();
+		this.admittedReads++;
+		try {
+			return await read();
+		} finally {
+			if (--this.admittedReads === 0) this.readsDrained?.();
+		}
+	}
+
+	private async readConversation(id: ConversationId): Promise<ConversationRecord | undefined> {
+		const row = await this.db.get<JsonRow>(`SELECT record FROM ${this.schema.tables.conversations} WHERE id = ?`, id);
+		return row === undefined ? undefined : parseJson<ConversationRecord>(row.record);
+	}
+
+	private async materializeDocument(
+		executor: SqliteExecutor,
+		id: DocumentId,
+		at: DocumentPoint,
+	): Promise<StoredDocument | undefined> {
+		const row = await executor.get<JsonRow>(`SELECT record FROM ${this.schema.tables.documents} WHERE id = ?`, id);
+		if (row === undefined) return undefined;
+		const record = parseJson<DocumentRecord>(row.record);
+		if (at !== "current" && isCurrentOnly(record)) {
+			throw new StorageRequestError(`Document ${id} does not retain historical content`);
+		}
+		if (!isAliveAt(record, at)) return undefined;
+		const upper = at === "current" ? Number.MAX_SAFE_INTEGER : at;
+		const base = await executor.get<RevisionRow>(
+			`SELECT seq, kind, version, content FROM ${this.schema.tables.documentRevisions}
+				WHERE document_id = ? AND kind = 'base' AND seq <= ? ORDER BY seq DESC LIMIT 1`,
+			id,
+			upper,
+		);
+		if (base === undefined) throw new Error(`Document ${id} is missing a required base`);
+		let value = parseJson<JsonObject>(base.content);
+		const tail = await executor.all<RevisionRow>(
+			`SELECT seq, kind, version, content FROM ${this.schema.tables.documentRevisions}
+				WHERE document_id = ? AND seq > ? AND seq <= ? ORDER BY seq`,
+			id,
+			base.seq,
+			upper,
+		);
+		for (const revision of tail) {
+			if (revision.kind !== "delta" || revision.version !== base.version) {
+				throw new Error(`Document ${id} crosses a stored version boundary without a base`);
+			}
+			value = apply(value, parseJson<readonly Op[]>(revision.content)) as JsonObject;
+		}
+		return { record, version: base.version, value, deltasSinceBase: tail.length };
+	}
+
+	private candidateNextId(writes: readonly StorageWrite[]): number {
+		let nextId = this.nextId;
+		for (const write of writes) {
+			const id = writeId(write);
+			if (id !== undefined) nextId = Math.max(nextId, id + 1);
+		}
+		return nextId;
+	}
+
+	private async checkGlobalIds(executor: SqliteExecutor, writes: readonly StorageWrite[]): Promise<void> {
+		const claimed = new Map<Id<string>, TableName>();
+		for (const write of writes) {
+			if (write.type === "document.change" || write.type === "document.retire") continue;
+			const document = write.type === "document.create" || write.type === "document.copy";
+			const table: TableName = document ? "document" : write.type;
+			const id = document ? write.record.id : write.value.id;
+			const existing = (
+				await executor.get<RecordIdRow>(`SELECT record_type FROM ${this.schema.tables.recordIds} WHERE id = ?`, id)
+			)?.record_type;
+			const earlier = claimed.get(id);
+			if (table === "conversation" || table === "entry" || table === "document") {
+				if (existing !== undefined) throw new Error(`ID ${id} already belongs to ${existing}`);
+				if (earlier !== undefined) throw new Error(`ID ${id} is written more than once`);
+			} else {
+				if (existing !== undefined && existing !== table)
+					throw new Error(`ID ${id} already belongs to ${existing}`);
+				if (earlier !== undefined && earlier !== table) throw new Error(`ID ${id} is written as two record types`);
+			}
+			claimed.set(id, table);
+		}
+	}
+
+	private prepareDocumentActions(writes: readonly StorageWrite[]): Map<DocumentId, DocumentAction> {
+		const actions = new Map<DocumentId, DocumentAction>();
+		for (const write of writes) {
+			if (
+				write.type !== "document.create" &&
+				write.type !== "document.copy" &&
+				write.type !== "document.change" &&
+				write.type !== "document.retire"
+			) {
+				continue;
+			}
+			const id = write.type === "document.create" || write.type === "document.copy" ? write.record.id : write.id;
+			let action = actions.get(id);
+			if (action === undefined) {
+				action = { retire: false };
+				actions.set(id, action);
+			}
+			switch (write.type) {
+				case "document.create":
+					if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) {
+						throw new Error(`Document ${id} has more than one content command`);
+					}
+					action.create = write.record;
+					action.content = write.content;
+					break;
+				case "document.copy":
+					if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) {
+						throw new Error(`Document ${id} has more than one content command`);
+					}
+					action.create = write.record;
+					action.copy = write.source;
+					break;
+				case "document.change":
+					if (action.content !== undefined || action.copy !== undefined) {
+						throw new Error(`Document ${id} has more than one content command`);
+					}
+					action.content = write.content;
+					break;
+				case "document.retire":
+					if (action.retire) throw new Error(`Document ${id} is retired more than once`);
+					action.retire = true;
+					break;
+			}
+		}
+		return actions;
+	}
+
+	private async checkDocumentActions(
+		executor: SqliteExecutor,
+		actions: ReadonlyMap<DocumentId, DocumentAction>,
+	): Promise<void> {
+		const liveCounts = new Map<string, number>();
+		for (const [id, action] of actions) {
+			if (action.copy !== undefined && actions.has(action.copy.id)) {
+				throw new Error(`Document copy ${id} source is changed in the copy batch`);
+			}
+			const row = await executor.get<JsonRow>(`SELECT record FROM ${this.schema.tables.documents} WHERE id = ?`, id);
+			const existing = row === undefined ? undefined : parseJson<DocumentRecord>(row.record);
+			if (action.create === undefined && existing === undefined) throw new Error(`Unknown document: ${id}`);
+			if (action.create !== undefined && existing !== undefined) throw new Error(`Document ${id} already exists`);
+			if (existing?.retiredAt !== undefined) throw new Error(`Document ${id} is retired`);
+			if (action.content?.kind === "delta") {
+				const previous = await executor.get<{ readonly version: number }>(
+					`SELECT version FROM ${this.schema.tables.documentRevisions} WHERE document_id = ? ORDER BY seq DESC LIMIT 1`,
+					id,
+				);
+				if (previous === undefined) throw new Error(`Document ${id} delta has no base`);
+				if (previous.version !== action.content.version) {
+					throw new Error(`Document ${id} version transition requires a base`);
+				}
+			}
+			const record = action.create ?? existing!;
+			const key = addressKey(record);
+			let live = liveCounts.get(key);
+			if (live === undefined) live = (await this.currentDocumentId(executor, record)) === undefined ? 0 : 1;
+			if (action.retire && existing !== undefined) live--;
+			if (action.create !== undefined && !action.retire) live++;
+			liveCounts.set(key, live);
+		}
+		for (const live of liveCounts.values()) {
+			if (live > 1) throw new Error("Document address already has a current incarnation");
+		}
+	}
+
+	private async currentDocumentId(
+		executor: SqliteExecutor,
+		address: DocumentAddress | DocumentCreate | DocumentRecord,
+	): Promise<DocumentId | undefined> {
+		const parts = addressParts(address);
+		const id = (
+			await executor.get<IdRow>(
+				`SELECT id FROM ${this.schema.tables.documents}
+				WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ? AND retired_at IS NULL
+				LIMIT 1`,
+				parts.kind,
+				parts.scopeKind,
+				parts.ownerId,
+				parts.family,
+				parts.keyValue,
+			)
+		)?.id;
+		return id === undefined ? undefined : idFromNumber<DocumentId>(id);
+	}
+
+	private async applyTableWrite(executor: SqliteExecutor, write: StorageWrite, seq: Seq): Promise<void> {
+		switch (write.type) {
+			case "conversation":
+				await this.claimId(executor, write.value.id, "conversation");
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.conversations} (id, owner_conversation_id, owner_task_id, record) VALUES (?, ?, ?, ?)`,
+					write.value.id,
+					write.value.owner?.conversationId ?? null,
+					write.value.owner?.taskId ?? null,
+					encodeJson(write.value),
+				);
+				break;
+			case "entry":
+				await this.claimId(executor, write.value.id, "entry");
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.entries} (id, conversation_id, head, commit_seq, record) VALUES (?, ?, ?, ?, ?)`,
+					write.value.id,
+					write.value.conversationId,
+					write.value.head ?? null,
+					seq,
+					encodeJson(write.value),
+				);
+				break;
+			case "task":
+				await this.claimId(executor, write.value.id, "task");
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.tasks} (id, conversation_id, kind, status, abort_requested, background, record)
+						VALUES (?, ?, ?, ?, ?, ?, ?)
+						ON CONFLICT(id) DO UPDATE SET conversation_id = excluded.conversation_id, kind = excluded.kind,
+						status = excluded.status, abort_requested = excluded.abort_requested,
+						background = excluded.background, record = excluded.record`,
+					write.value.id,
+					write.value.conversationId,
+					encodeIndexedString(write.value.kind),
+					write.value.state.status,
+					write.value.abortRequested ? 1 : 0,
+					write.value.background ? 1 : 0,
+					encodeJson(write.value),
+				);
+				break;
+			case "submission":
+				await this.claimId(executor, write.value.id, "submission");
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.submissions} (id, conversation_id, request_id, status, record) VALUES (?, ?, ?, ?, ?)
+						ON CONFLICT(id) DO UPDATE SET conversation_id = excluded.conversation_id,
+						request_id = excluded.request_id, status = excluded.status, record = excluded.record`,
+					write.value.id,
+					write.value.conversationId,
+					write.value.requestId === undefined ? null : encodeIndexedString(write.value.requestId),
+					write.value.status,
+					encodeJson(write.value),
+				);
+				break;
+			case "document.create":
+			case "document.copy":
+			case "document.change":
+			case "document.retire":
+				break;
+		}
+	}
+
+	private async claimId(executor: SqliteExecutor, id: Id<string>, table: TableName): Promise<void> {
+		await executor.run(
+			`INSERT OR IGNORE INTO ${this.schema.tables.recordIds} (id, record_type) VALUES (?, ?)`,
+			id,
+			table,
+		);
+	}
+
+	private async applyDocumentActions(
+		executor: SqliteExecutor,
+		actions: ReadonlyMap<DocumentId, DocumentAction>,
+		seq: Seq,
+	): Promise<void> {
+		for (const [id, action] of actions) {
+			let content = action.content;
+			if (action.copy !== undefined) {
+				try {
+					const stored = await this.materializeDocument(executor, action.copy.id, action.copy.at);
+					if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`);
+					const create = action.create!;
+					if (
+						stored.record.scope.kind !== "conversation" ||
+						create.scope.kind !== "conversation" ||
+						stored.record.kind !== create.kind ||
+						stored.record.key !== create.key ||
+						stored.record.history !== create.history ||
+						stored.record.fork !== create.fork
+					) {
+						throw new Error(`Fork source document ${action.copy.id} does not match the copied record`);
+					}
+					content = { kind: "base", version: stored.version, value: stored.value };
+				} catch (error) {
+					throw new Error(`Document copy ${id} was rejected`, { cause: error });
+				}
+			}
+			let record: DocumentRecord;
+			if (action.create !== undefined) {
+				record = {
+					...action.create,
+					createdAt: seq,
+					...(action.retire ? { retiredAt: seq } : {}),
+				};
+				const parts = addressParts(record);
+				await this.claimId(executor, id, "document");
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.documents}
+						(id, kind, family, key_value, scope_kind, owner_id, created_at, retired_at, record)
+						VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+					id,
+					parts.kind,
+					parts.family,
+					parts.keyValue,
+					parts.scopeKind,
+					parts.ownerId,
+					seq,
+					action.retire ? seq : null,
+					encodeJson(record),
+				);
+			} else {
+				const row = (await executor.get<JsonRow>(
+					`SELECT record FROM ${this.schema.tables.documents} WHERE id = ?`,
+					id,
+				))!;
+				record = parseJson<DocumentRecord>(row.record);
+			}
+
+			if (content !== undefined) {
+				if (content.kind === "base" && isCurrentOnly(record)) {
+					await executor.run(`DELETE FROM ${this.schema.tables.documentRevisions} WHERE document_id = ?`, id);
+				}
+				const encodedContent = content.kind === "base" ? encodeJson(content.value) : encodeJson(content.ops);
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.documentRevisions} (document_id, seq, kind, version, content) VALUES (?, ?, ?, ?, ?)`,
+					id,
+					seq,
+					content.kind,
+					content.version,
+					encodedContent,
+				);
+			}
+
+			if (action.retire) {
+				if (action.create === undefined) {
+					record = { ...record, retiredAt: seq };
+					await executor.run(
+						`UPDATE ${this.schema.tables.documents} SET retired_at = ?, record = ? WHERE id = ?`,
+						seq,
+						encodeJson(record),
+						id,
+					);
+				}
+				if (isCurrentOnly(record)) {
+					await executor.run(`DELETE FROM ${this.schema.tables.documentRevisions} WHERE document_id = ?`, id);
+				}
+			}
+		}
+	}
+
+	private assertOpen(): void {
+		if (this.closed) throw new Error("SqliteStorage is closed");
+	}
+}

third_party/pi/packages/durable/src/tasks.ts

@@ -0,0 +1,8 @@
+import type { Task, TaskDefinition } from "./types.ts";
+
+/** Define an executable task. Register it in the registry so a Harness can run tasks of its kind. */
+export function defineTask<I, S extends { phase: string }, R, H extends object = object>(
+	definition: TaskDefinition<I, S, R, H>,
+): Task<I, S, R, H> {
+	return { definition };
+}

third_party/pi/packages/durable/src/testing/assertions.ts

@@ -0,0 +1,39 @@
+import type { StorageConformanceAssertions } from "./types.ts";
+
+export type ExpectLike = (actual: unknown, message?: string) => unknown;
+
+type ExpectResult = {
+	toBe(expected: unknown): unknown;
+	toBeGreaterThan(expected: number): unknown;
+	toEqual(expected: unknown): unknown;
+	toMatchObject(expected: unknown): unknown;
+	toBeTruthy(): unknown;
+	readonly rejects: {
+		toThrow(expected?: string | RegExp): unknown;
+	};
+};
+
+/** Adapts a Vitest/Jest-compatible `expect` function without importing either runner. */
+export function createExpectAssertions(expect: ExpectLike): StorageConformanceAssertions {
+	const result = (actual: unknown, message?: string): ExpectResult => expect(actual, message) as ExpectResult;
+	return {
+		ok(value, message) {
+			result(value, message).toBeTruthy();
+		},
+		strictEqual(actual, expected) {
+			result(actual).toBe(expected);
+		},
+		deepEqual(actual, expected) {
+			result(actual).toEqual(expected);
+		},
+		partialDeepEqual(actual, expected) {
+			result(actual).toMatchObject(expected);
+		},
+		greaterThan(actual, expected) {
+			result(actual).toBeGreaterThan(expected);
+		},
+		async rejects(operation, messageIncludes) {
+			await result(operation).rejects.toThrow(messageIncludes);
+		},
+	};
+}

third_party/pi/packages/durable/src/testing/env-conformance.ts

@@ -0,0 +1,633 @@
+import type { Context } from "@earendil-works/chord";
+import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/context";
+import {
+	type ExecutionEnv,
+	type FileInfo,
+	getOrThrow,
+	type ListFilesOptions,
+	type Result,
+	type ShellOutputInfo,
+	type WatchChange,
+	type WatchTarget,
+} from "../env/index.ts";
+import type { EnvConformanceCase, EnvConformanceOptions } from "./types.ts";
+
+const context = BACKGROUND_CONTEXT;
+const decoder = new TextDecoder();
+const encoder = new TextEncoder();
+
+type EnvTest = (env: ExecutionEnv) => Promise<void>;
+
+function abortedContext(): Context {
+	const controller = new AbortController();
+	controller.abort();
+	return withAbortSignal(controller.signal, BACKGROUND_CONTEXT);
+}
+
+function errorCode(result: Result<unknown, { code: string }>): string | undefined {
+	return result.ok ? undefined : result.error.code;
+}
+
+async function readAll(
+	env: ExecutionEnv,
+	path: string,
+	maxEntries: number,
+): Promise<{ pages: FileInfo[][]; done: boolean }> {
+	const reader = getOrThrow(await env.openDirReader(path, context));
+	const pages: FileInfo[][] = [];
+	try {
+		for (let page = 0; page < 1000; page++) {
+			const next = getOrThrow(await reader.next(maxEntries, context));
+			pages.push(next.entries);
+			if (next.done) return { pages, done: true };
+		}
+		return { pages, done: false };
+	} finally {
+		await reader.close(context);
+	}
+}
+
+/** A listing as `<kind letter> <path>` lines (`f`, `d`, `l`), with its truncation. */
+async function listed(
+	env: ExecutionEnv,
+	path: string,
+	options?: ListFilesOptions,
+): Promise<{ entries: string[]; truncated: boolean }> {
+	if (env.listFiles === undefined) throw new Error("The environment does not implement listFiles");
+	const listing = getOrThrow(await env.listFiles(path, options, context));
+	const letter = { file: "f", directory: "d", symlink: "l" } as const;
+	return {
+		entries: listing.entries.map((entry) => `${letter[entry.kind]} ${entry.path}`),
+		truncated: listing.truncated,
+	};
+}
+
+/** Whether a change reports `path`: an overflow, or a reported path at or above it. */
+function covers(change: WatchChange, path: string): boolean {
+	if ("overflow" in change) return true;
+	if (!("paths" in change)) return false;
+	return change.paths.some(
+		(reported) => path === reported || path.startsWith(`${reported}/`) || path.startsWith(`${reported}\\`),
+	);
+}
+
+/** Watch `targets` while `run` changes files, with a helper that waits for a change reporting a path. */
+async function watching(
+	env: ExecutionEnv,
+	targets: readonly WatchTarget[],
+	run: (helpers: {
+		readonly changes: WatchChange[];
+		/** Wait up to three seconds for a change, after the call, that reports `path`. */
+		expectChange(path: string, change: () => Promise<void>): Promise<void>;
+		absolute(path: string): Promise<string>;
+	}) => Promise<void>,
+): Promise<void> {
+	const changes: WatchChange[] = [];
+	const watcher = getOrThrow(await env.watch(targets, (change) => changes.push(change), context));
+	try {
+		const absolute = async (path: string) => getOrThrow(await env.absolutePath(path, context));
+		await run({
+			changes,
+			absolute,
+			async expectChange(path, change) {
+				const target = await absolute(path);
+				const from = changes.length;
+				await change();
+				const deadline = Date.now() + 3000;
+				while (!changes.slice(from).some((entry) => covers(entry, target))) {
+					if (Date.now() > deadline) {
+						throw new Error(`No change reported ${target}; got ${JSON.stringify(changes.slice(from))}`);
+					}
+					await new Promise((resolve) => setTimeout(resolve, 20));
+				}
+			},
+		});
+	} finally {
+		await watcher.close(context);
+	}
+}
+
+/**
+ * Creates runner-independent cases for an `ExecutionEnv`. `withEnv` must call and await its callback exactly once per
+ * case with an environment whose `cwd` is a fresh, empty, writable directory.
+ */
+export function createEnvConformance(options: EnvConformanceOptions): readonly EnvConformanceCase[] {
+	const assert = options.assertions;
+	const shell = options.shell ?? ["sh", "-c"];
+	const symlinks = options.symlinks ?? true;
+	const createCase = (name: string, test: EnvTest, timeoutMs?: number): EnvConformanceCase => ({
+		name,
+		...(timeoutMs === undefined ? {} : { timeoutMs }),
+		run: () => options.withEnv(test),
+	});
+	// Watch cases wait up to three seconds per step, longer than test runners allow by default.
+	const watchCase = (name: string, test: EnvTest): EnvConformanceCase => createCase(name, test, 30_000);
+	const execCollect = async (env: ExecutionEnv, command: string | readonly string[], cwd?: string) => {
+		const output: Record<ShellOutputInfo["stream"], string> = { stdout: "", stderr: "" };
+		const result = await env.exec(
+			command,
+			{
+				...(cwd === undefined ? {} : { cwd }),
+				onOutput: (text, _context, info) => {
+					output[info.stream] += text;
+				},
+			},
+			context,
+		);
+		return { result, ...output };
+	};
+
+	const cases: EnvConformanceCase[] = [
+		createCase("binary reader reads byte ranges of the opened file", async (env) => {
+			getOrThrow(await env.writeFile("data.txt", "hello world", context));
+			const reader = getOrThrow(await env.openBinaryReader("data.txt", undefined, context));
+			const info = getOrThrow(await reader.info(context));
+			assert.partialDeepEqual(info, { name: "data.txt", kind: "file", size: 11 });
+			assert.strictEqual(decoder.decode(getOrThrow(await reader.read(0, 5, context))), "hello");
+			assert.strictEqual(decoder.decode(getOrThrow(await reader.read(6, 100, context))), "world");
+			assert.strictEqual(getOrThrow(await reader.read(11, 4, context)).length, 0);
+			assert.strictEqual(getOrThrow(await reader.read(50, 1, context)).length, 0);
+			assert.strictEqual(getOrThrow(await reader.read(3, 0, context)).length, 0);
+			assert.strictEqual(errorCode(await reader.read(-1, 1, context)), "invalid");
+			assert.strictEqual(errorCode(await reader.read(0, 1.5, context)), "invalid");
+			assert.strictEqual(errorCode(await reader.read(0, 1, abortedContext())), "aborted");
+			await reader.close(context);
+			await reader.close(context);
+			assert.strictEqual(errorCode(await reader.read(0, 1, context)), "invalid");
+			assert.strictEqual(errorCode(await reader.info(context)), "invalid");
+		}),
+
+		createCase("binary reader scans lines like decoding the whole file", async (env) => {
+			// A byte-order mark, an invalid sequence before a newline, an empty line, a later U+FEFF, and no final newline.
+			const bytes = Uint8Array.from([
+				0xef, 0xbb, 0xbf, 0x61, 0x0a, 0xe2, 0x82, 0x0a, 0x0a, 0xef, 0xbb, 0xbf, 0x62, 0x0a, 0xc3, 0xa9,
+			]);
+			getOrThrow(await env.writeFile("lines.txt", bytes, context));
+			const lines = new TextDecoder().decode(bytes).split("\n");
+			const reader = getOrThrow(await env.openBinaryReader("lines.txt", undefined, context));
+			try {
+				const ranges: [number, number | undefined][] = [
+					[0, undefined],
+					[0, 1],
+					[1, 3],
+					[2, 3],
+					[3, undefined],
+					[4, 9],
+				];
+				for (const [startLine, endLine] of ranges) {
+					const scan = getOrThrow(
+						await reader.scanLines({ startLine, ...(endLine === undefined ? {} : { endLine }) }, context),
+					);
+					const selected = lines.slice(startLine, endLine);
+					const range = (from: number, to: number) =>
+						new TextDecoder("utf-8", { ignoreBOM: from > 0 }).decode(bytes.subarray(from, to));
+					assert.strictEqual(scan.newlines, lines.length - 1);
+					assert.strictEqual(range(scan.start, scan.end), selected.join("\n"));
+					assert.strictEqual(scan.selectedBytes, encoder.encode(selected.join("\n")).length);
+					assert.strictEqual(range(scan.start, scan.firstLineEnd), lines[startLine]);
+					assert.strictEqual(scan.firstLineBytes, encoder.encode(lines[startLine]).length);
+				}
+				assert.partialDeepEqual(getOrThrow(await reader.scanLines({ startLine: 9 }, context)), {
+					start: bytes.length,
+					end: bytes.length,
+					selectedBytes: 0,
+				});
+				assert.strictEqual(errorCode(await reader.scanLines({ startLine: 2, endLine: 2 }, context)), "invalid");
+			} finally {
+				await reader.close(context);
+			}
+		}),
+
+		createCase("binary reader keeps reading the file it opened after a rename", async (env) => {
+			getOrThrow(await env.writeFile("a.txt", "one", context));
+			const reader = getOrThrow(await env.openBinaryReader("a.txt", undefined, context));
+			try {
+				getOrThrow(await env.renameFile("a.txt", "b.txt", context));
+				getOrThrow(await env.writeFile("a.txt", "two", context));
+				assert.strictEqual(decoder.decode(getOrThrow(await reader.read(0, 10, context))), "one");
+			} finally {
+				await reader.close(context);
+			}
+		}),
+
+		createCase("binary reader refuses directories, missing files and aborted opens", async (env) => {
+			getOrThrow(await env.createDir("dir", undefined, context));
+			getOrThrow(await env.writeFile("file.txt", "x", context));
+			assert.strictEqual(errorCode(await env.openBinaryReader("dir", undefined, context)), "is_directory");
+			assert.strictEqual(errorCode(await env.openBinaryReader("missing.txt", undefined, context)), "not_found");
+			assert.strictEqual(errorCode(await env.openBinaryReader("file.txt", undefined, abortedContext())), "aborted");
+		}),
+
+		createCase("directory reader pages every entry exactly once", async (env) => {
+			const names = ["a.txt", "b.txt", "c.txt", "d.txt", "e.txt"];
+			for (const name of names) getOrThrow(await env.writeFile(name, name, context));
+			getOrThrow(await env.createDir("sub", undefined, context));
+			const { pages, done } = await readAll(env, ".", 2);
+			assert.ok(done, "directory reader reached the end");
+			for (const page of pages) assert.ok(page.length <= 2, "page within maxEntries");
+			const entries = pages.flat();
+			assert.deepEqual(entries.map((entry) => entry.name).sort(), [...names, "sub"].sort());
+			assert.strictEqual(entries.find((entry) => entry.name === "sub")?.kind, "directory");
+			assert.partialDeepEqual(
+				entries.find((entry) => entry.name === "a.txt"),
+				{ kind: "file", size: 5 },
+			);
+		}),
+
+		createCase("directory reader reports the end and refuses use after close", async (env) => {
+			getOrThrow(await env.createDir("empty", undefined, context));
+			const reader = getOrThrow(await env.openDirReader("empty", context));
+			assert.deepEqual(getOrThrow(await reader.next(10, context)), { entries: [], done: true });
+			assert.deepEqual(getOrThrow(await reader.next(10, context)), { entries: [], done: true });
+			assert.strictEqual(errorCode(await reader.next(0, context)), "invalid");
+			assert.strictEqual(errorCode(await reader.next(1, abortedContext())), "aborted");
+			await reader.close(context);
+			await reader.close(context);
+			assert.strictEqual(errorCode(await reader.next(1, context)), "invalid");
+		}),
+
+		createCase("directory reader refuses missing paths and files", async (env) => {
+			getOrThrow(await env.writeFile("file.txt", "x", context));
+			assert.strictEqual(errorCode(await env.openDirReader("missing", context)), "not_found");
+			const file = await env.openDirReader("file.txt", context);
+			assert.strictEqual(errorCode(file), "not_directory");
+			assert.strictEqual(errorCode(await env.openDirReader(".", abortedContext())), "aborted");
+		}),
+
+		createCase("directory reader skips entries removed during enumeration", async (env) => {
+			getOrThrow(await env.createDir("dir", undefined, context));
+			for (const name of ["x", "y", "z"]) getOrThrow(await env.writeFile(`dir/${name}`, name, context));
+			const reader = getOrThrow(await env.openDirReader("dir", context));
+			try {
+				for (const name of ["x", "y", "z"]) getOrThrow(await env.remove(`dir/${name}`, undefined, context));
+				const entries: FileInfo[] = [];
+				for (let page = 0; page < 10; page++) {
+					const next = getOrThrow(await reader.next(10, context));
+					entries.push(...next.entries);
+					if (next.done) break;
+				}
+				assert.deepEqual(entries, []);
+			} finally {
+				await reader.close(context);
+			}
+		}),
+
+		watchCase("watch reports a missing file's creation, changes, replacement and removal", async (env) => {
+			await watching(env, [{ path: "AGENTS.md" }], async ({ expectChange }) => {
+				await expectChange("AGENTS.md", async () => {
+					getOrThrow(await env.writeFile("AGENTS.md", "one", context));
+				});
+				await expectChange("AGENTS.md", async () => {
+					getOrThrow(await env.writeFile("AGENTS.md", "two!", context));
+				});
+				// Editors replace a file by renaming a new one over it.
+				await expectChange("AGENTS.md", async () => {
+					getOrThrow(await env.writeFile("AGENTS.md.tmp", "three", context));
+					getOrThrow(await env.renameFile("AGENTS.md.tmp", "AGENTS.md", context));
+				});
+				await expectChange("AGENTS.md", async () => {
+					getOrThrow(await env.writeFile("AGENTS.md", "four", context));
+				});
+				await expectChange("AGENTS.md", async () => {
+					getOrThrow(await env.remove("AGENTS.md", undefined, context));
+				});
+			});
+		}),
+
+		watchCase("watch reports a missing target whose ancestors are created", async (env) => {
+			await watching(env, [{ path: "a/b/c/AGENTS.md" }], async ({ expectChange }) => {
+				await expectChange("a/b/c/AGENTS.md", async () => {
+					getOrThrow(await env.writeFile("a/b/c/AGENTS.md", "x", context));
+				});
+			});
+		}),
+
+		watchCase("watch follows directories created together with their contents", async (env) => {
+			getOrThrow(await env.createDir("skills", undefined, context));
+			await watching(env, [{ path: "skills", recursive: true }], async ({ expectChange }) => {
+				// Written before any watcher on the new directories can exist.
+				await expectChange("skills/a/b/SKILL.md", async () => {
+					getOrThrow(await env.writeFile("skills/a/b/SKILL.md", "one", context));
+				});
+				await expectChange("skills/a/b/SKILL.md", async () => {
+					getOrThrow(await env.writeFile("skills/a/b/SKILL.md", "two!", context));
+				});
+				await expectChange("skills/a/b/c/SKILL.md", async () => {
+					getOrThrow(await env.writeFile("skills/a/b/c/SKILL.md", "deeper", context));
+				});
+			});
+		}),
+
+		watchCase("watch keeps watching a path whose parent is renamed and recreated", async (env) => {
+			getOrThrow(await env.writeFile("proj/.pi/skills/x.md", "x", context));
+			await watching(env, [{ path: "proj/.pi/skills", recursive: true }], async ({ expectChange }) => {
+				await expectChange("proj/.pi/skills", async () => {
+					getOrThrow(await env.renameFile("proj/.pi", "proj/old", context));
+				});
+				await expectChange("proj/.pi/skills/y.md", async () => {
+					getOrThrow(await env.writeFile("proj/.pi/skills/y.md", "y", context));
+				});
+				await expectChange("proj/.pi/skills/y.md", async () => {
+					getOrThrow(await env.writeFile("proj/.pi/skills/y.md", "yy", context));
+				});
+			});
+		}),
+
+		watchCase("watch skips excluded entries and reports a rename out of them", async (env) => {
+			getOrThrow(await env.createDir("skills", undefined, context));
+			const targets: WatchTarget[] = [
+				{ path: "skills", recursive: true, exclude: { hidden: true, names: ["node_modules"] } },
+			];
+			await watching(env, targets, async ({ changes, expectChange, absolute }) => {
+				getOrThrow(await env.writeFile("skills/node_modules/dep/SKILL.md", "dep", context));
+				getOrThrow(await env.writeFile("skills/.SKILL.md.tmp", "draft", context));
+				await expectChange("skills/SKILL.md", async () => {
+					getOrThrow(await env.renameFile("skills/.SKILL.md.tmp", "skills/SKILL.md", context));
+				});
+				const hidden = [await absolute("skills/node_modules"), await absolute("skills/.SKILL.md.tmp")];
+				for (const change of changes) {
+					if (!("paths" in change)) continue;
+					for (const path of change.paths) {
+						assert.ok(
+							!hidden.some((excluded) => path === excluded || path.startsWith(excluded)),
+							`excluded ${path}`,
+						);
+					}
+				}
+			});
+		}),
+
+		watchCase("watch keeps recursive coverage where a non-recursive target overlaps", async (env) => {
+			getOrThrow(await env.writeFile("skills/a/one.md", "one", context));
+			const targets: WatchTarget[] = [{ path: "skills" }, { path: "skills", recursive: true }];
+			await watching(env, targets, async ({ expectChange }) => {
+				await expectChange("skills/a/two.md", async () => {
+					getOrThrow(await env.writeFile("skills/a/two.md", "two", context));
+				});
+			});
+		}),
+
+		watchCase("watch follows a directory replaced at the same path", async (env) => {
+			getOrThrow(await env.writeFile("skills/a/x.md", "x", context));
+			await watching(env, [{ path: "skills", recursive: true }], async ({ expectChange }) => {
+				await expectChange("skills/a", async () => {
+					getOrThrow(await env.renameFile("skills/a", "skills-old", context));
+					getOrThrow(await env.createDir("skills/a", undefined, context));
+				});
+				await expectChange("skills/a/y.md", async () => {
+					getOrThrow(await env.writeFile("skills/a/y.md", "y", context));
+				});
+				await expectChange("skills/a/y.md", async () => {
+					getOrThrow(await env.writeFile("skills/a/y.md", "yy", context));
+				});
+			});
+		}),
+
+		watchCase("watch stops reporting once closed", async (env) => {
+			const changes: WatchChange[] = [];
+			const watcher = getOrThrow(await env.watch([{ path: "file.txt" }], (change) => changes.push(change), context));
+			assert.ok(watcher.mode === "native" || watcher.mode === "polling", "watcher reports its mode");
+			await watcher.close(context);
+			await watcher.close(context);
+			getOrThrow(await env.writeFile("file.txt", "x", context));
+			await new Promise((resolve) => setTimeout(resolve, 300));
+			assert.deepEqual(changes, []);
+		}),
+
+		createCase("argv exec passes arguments to the program without shell parsing", async (env) => {
+			const hostile = "it's $(touch pwned) `touch pwned` *; touch pwned";
+			const { result, stdout } = await execCollect(env, [
+				...shell,
+				'printf "%s|%s" "$1" "$2"',
+				"argv0",
+				hostile,
+				"a b",
+			]);
+			assert.strictEqual(getOrThrow(result).exitCode, 0);
+			assert.strictEqual(stdout, `${hostile}|a b`);
+			assert.strictEqual(getOrThrow(await env.exists("pwned", context)), false);
+		}),
+
+		createCase("exec reports the stream of every chunk in both forms", async (env) => {
+			const script = "printf out; printf err >&2; printf more";
+			const argv = await execCollect(env, [...shell, script]);
+			assert.strictEqual(getOrThrow(argv.result).exitCode, 0);
+			assert.strictEqual(argv.stdout, "outmore");
+			assert.strictEqual(argv.stderr, "err");
+			const string = await execCollect(env, script);
+			assert.strictEqual(getOrThrow(string.result).exitCode, 0);
+			assert.strictEqual(string.stdout, "outmore");
+			assert.strictEqual(string.stderr, "err");
+		}),
+
+		createCase("argv exec honors cwd and exit codes", async (env) => {
+			getOrThrow(await env.createDir("sub", undefined, context));
+			const made = await execCollect(env, [...shell, "printf x > made.txt; exit 3"], "sub");
+			assert.strictEqual(getOrThrow(made.result).exitCode, 3);
+			assert.strictEqual(getOrThrow(await env.readTextFile("sub/made.txt", context)), "x");
+		}),
+
+		createCase("argv exec reports missing programs and empty argv as spawn errors", async (env) => {
+			assert.strictEqual(
+				errorCode(await env.exec(["pi-durable-conformance-missing-program"], undefined, context)),
+				"spawn_error",
+			);
+			assert.strictEqual(errorCode(await env.exec([], undefined, context)), "spawn_error");
+		}),
+
+		createCase("windowed exec keeps the exact tail and counts what it skips", async (env) => {
+			const lines = 2000;
+			const window = { maxBytes: 200, maxLines: 5, minIntervalMs: 0, bytesPerSecond: 1_000_000_000 };
+			let bytes = 0;
+			let newlines = 0;
+			let tail = "";
+			const result = await env.exec(
+				[...shell, `i=0; while [ $i -lt ${lines} ]; do echo line-$i; i=$((i+1)); done`],
+				{
+					window,
+					onOutput: (text, _context, info) => {
+						if (info.skipped !== undefined) {
+							bytes += info.skipped.bytes;
+							newlines += info.skipped.newlines;
+							const after = encoder.encode(text).length;
+							const afterNewlines = text.split("\n").length - 1;
+							assert.ok(
+								after > window.maxBytes || afterNewlines > window.maxLines,
+								"a skip is followed by more than the window",
+							);
+							tail = "";
+						}
+						bytes += encoder.encode(text).length;
+						newlines += text.split("\n").length - 1;
+						tail += text;
+					},
+				},
+				context,
+			);
+			assert.strictEqual(getOrThrow(result).exitCode, 0);
+			const expected = Array.from({ length: lines }, (_, index) => `line-${index}\n`);
+			assert.strictEqual(bytes, expected.join("").length);
+			assert.strictEqual(newlines, lines);
+			assert.ok(tail.endsWith(expected.slice(-window.maxLines).join("")), "the delivered output ends with the tail");
+		}),
+
+		createCase("argv exec distinguishes timeout from abort", async (env) => {
+			const timedOut = await env.exec([...shell, "sleep 2"], { timeout: 0.1 }, context);
+			assert.strictEqual(errorCode(timedOut), "timeout");
+			const controller = new AbortController();
+			const running = env.exec([...shell, "sleep 2"], undefined, withAbortSignal(controller.signal, context));
+			setTimeout(() => controller.abort(), 100);
+			assert.strictEqual(errorCode(await running), "aborted");
+		}),
+	];
+
+	if (options.listFiles) {
+		const writeAll = async (env: ExecutionEnv, files: readonly string[]) => {
+			for (const file of files) getOrThrow(await env.writeFile(file, "", context));
+		};
+		cases.push(
+			createCase("listFiles lists files and directories depth first in name order", async (env) => {
+				await writeAll(env, ["b.txt", "a/z.txt", "a/b/c.txt", ".hidden/x", "A.txt"]);
+				getOrThrow(await env.createDir("empty", undefined, context));
+				assert.deepEqual(await listed(env, "."), {
+					entries: [
+						"d .hidden",
+						"f .hidden/x",
+						"f A.txt",
+						"d a",
+						"d a/b",
+						"f a/b/c.txt",
+						"f a/z.txt",
+						"f b.txt",
+						"d empty",
+					],
+					truncated: false,
+				});
+				assert.deepEqual((await listed(env, ".", { hidden: false })).entries, [
+					"f A.txt",
+					"d a",
+					"d a/b",
+					"f a/b/c.txt",
+					"f a/z.txt",
+					"f b.txt",
+					"d empty",
+				]);
+				const expected = { entries: ["d b", "f b/c.txt", "f z.txt"], truncated: false };
+				assert.deepEqual(await listed(env, "a"), expected);
+				assert.deepEqual(await listed(env, getOrThrow(await env.absolutePath("a", context))), expected);
+				assert.deepEqual(await listed(env, "empty"), { entries: [], truncated: false });
+			}),
+
+			createCase("listFiles skips what git ignores in work trees, .git, and excluded names", async (env) => {
+				await writeAll(env, ["keep.txt", "out/x.js", "sub/keep.txt", "sub/drop.log", "node_modules/m/i.js"]);
+				getOrThrow(await env.writeFile(".gitignore", "out/\n*.log\n", context));
+				getOrThrow(await env.writeFile("sub/.gitignore", "!keep.txt\n", context));
+				// Not a work tree yet: `.gitignore` files have no effect.
+				assert.ok((await listed(env, ".")).entries.includes("f out/x.js"));
+				getOrThrow(await env.writeFile(".git/HEAD", "ref: refs/heads/main\n", context));
+				getOrThrow(await env.writeFile(".git/info/exclude", "keep.txt\n", context));
+				assert.deepEqual((await listed(env, ".", { exclude: ["node_modules"] })).entries, [
+					"f .gitignore",
+					"d sub",
+					"f sub/.gitignore",
+					"f sub/keep.txt",
+				]);
+				// A directory inside the work tree applies its parents' rules.
+				assert.deepEqual((await listed(env, "sub")).entries, ["f .gitignore", "f keep.txt"]);
+				const all = (await listed(env, ".", { gitignore: false })).entries;
+				assert.ok(
+					all.includes("f out/x.js") && all.includes("f node_modules/m/i.js") && all.includes("f keep.txt"),
+				);
+				assert.ok(!all.some((entry) => entry.includes(".git/") || entry === "d .git"));
+			}),
+
+			createCase("listFiles stops at its bounds and reports truncated", async (env) => {
+				await writeAll(env, ["a", "b", "c"]);
+				assert.deepEqual(await listed(env, ".", { maxEntries: 2 }), { entries: ["f a", "f b"], truncated: true });
+				assert.deepEqual(await listed(env, ".", { maxEntries: 3 }), {
+					entries: ["f a", "f b", "f c"],
+					truncated: false,
+				});
+				// Each entry counts its path plus two bytes.
+				assert.deepEqual(await listed(env, ".", { maxBytes: 6 }), { entries: ["f a", "f b"], truncated: true });
+				assert.deepEqual(await listed(env, ".", { maxEntries: 0 }), { entries: [], truncated: true });
+				assert.strictEqual(errorCode(await env.listFiles!(".", { maxEntries: -1 }, context)), "invalid");
+				assert.strictEqual(errorCode(await env.listFiles!(".", { timeoutMs: 1.5 }, context)), "invalid");
+			}),
+
+			createCase("listFiles fails on missing directories, files, and aborts", async (env) => {
+				getOrThrow(await env.writeFile("file.txt", "", context));
+				assert.strictEqual(errorCode(await env.listFiles!("missing", undefined, context)), "not_found");
+				assert.strictEqual(errorCode(await env.listFiles!("file.txt", undefined, context)), "not_directory");
+				assert.strictEqual(errorCode(await env.listFiles!(".", undefined, abortedContext())), "aborted");
+			}),
+		);
+	}
+
+	if (symlinks && options.listFiles) {
+		cases.push(
+			createCase("listFiles lists symbolic links without following them", async (env) => {
+				getOrThrow(await env.writeFile("dir/inner.txt", "", context));
+				const linked = await env.exec(
+					[...shell, "ln -s dir link && ln -s missing dangling && ln -s .. dir/loop"],
+					undefined,
+					context,
+				);
+				assert.strictEqual(getOrThrow(linked).exitCode, 0);
+				assert.deepEqual((await listed(env, ".")).entries, [
+					"l dangling",
+					"d dir",
+					"f dir/inner.txt",
+					"l dir/loop",
+					"l link",
+				]);
+				// The listed directory itself may be a link.
+				assert.deepEqual((await listed(env, "link")).entries, ["f inner.txt", "l loop"]);
+			}),
+		);
+	}
+
+	if (symlinks) {
+		cases.push(
+			createCase("binary reader follows symlinks unless noFollow refuses the final one", async (env) => {
+				getOrThrow(await env.writeFile("target.txt", "target", context));
+				getOrThrow(await env.createDir("sub", undefined, context));
+				getOrThrow(await env.writeFile("sub/inner.txt", "inner", context));
+				const linked = await env.exec(
+					[...shell, "ln -s target.txt link.txt && ln -s sub dirlink"],
+					undefined,
+					context,
+				);
+				assert.strictEqual(getOrThrow(linked).exitCode, 0);
+
+				const followed = getOrThrow(await env.openBinaryReader("link.txt", undefined, context));
+				assert.strictEqual(decoder.decode(getOrThrow(await followed.read(0, 10, context))), "target");
+				await followed.close(context);
+
+				assert.strictEqual(
+					errorCode(await env.openBinaryReader("link.txt", { noFollow: true }, context)),
+					"invalid",
+				);
+
+				// Only the final component is refused; earlier symlinked directories still resolve.
+				const inner = getOrThrow(await env.openBinaryReader("dirlink/inner.txt", { noFollow: true }, context));
+				assert.strictEqual(decoder.decode(getOrThrow(await inner.read(0, 10, context))), "inner");
+				await inner.close(context);
+			}),
+
+			watchCase("watch reports changes to the file a watched symbolic link points to", async (env) => {
+				getOrThrow(await env.writeFile("data/real.md", "one", context));
+				getOrThrow(await env.createDir("config", undefined, context));
+				const linked = await env.exec([...shell, "ln -s ../data/real.md config/AGENTS.md"], undefined, context);
+				assert.strictEqual(getOrThrow(linked).exitCode, 0);
+				await watching(env, [{ path: "config/AGENTS.md" }], async ({ expectChange }) => {
+					await expectChange("config/AGENTS.md", async () => {
+						getOrThrow(await env.writeFile("data/real.md", "two!", context));
+					});
+				});
+			}),
+		);
+	}
+
+	return cases;
+}

third_party/pi/packages/durable/src/testing/index.ts

@@ -0,0 +1,27 @@
+export { createExpectAssertions, type ExpectLike } from "./assertions.ts";
+export { createEnvConformance } from "./env-conformance.ts";
+export { registerEnvConformance, registerStorageConformance, type StorageConformanceRunner } from "./runner.ts";
+export {
+	STORAGE_MEMORY_SCALES,
+	STORAGE_READ_BENCHMARKS,
+	STORAGE_WRITE_BENCHMARKS,
+	type StorageBenchmarkDataset,
+	type StorageBenchmarkScale,
+	type StorageReadBenchmark,
+	type StorageWriteBenchmark,
+	seedStorageBenchmark,
+	seedStorageWriteBenchmark,
+	storageBenchmarkPrimaryRecordCount,
+	TIMING_SCALE,
+} from "./storage-benchmark.ts";
+export { createStorageConformance } from "./storage-conformance.ts";
+export type {
+	EnvConformanceAssertions,
+	EnvConformanceCase,
+	EnvConformanceOptions,
+	EnvConformanceProvider,
+	StorageConformanceAssertions,
+	StorageConformanceCase,
+	StorageConformanceOptions,
+	StorageConformanceProvider,
+} from "./types.ts";

third_party/pi/packages/durable/src/testing/runner.ts

@@ -0,0 +1,38 @@
+import { createExpectAssertions, type ExpectLike } from "./assertions.ts";
+import { createEnvConformance } from "./env-conformance.ts";
+import { createStorageConformance } from "./storage-conformance.ts";
+import type { EnvConformanceOptions, EnvConformanceProvider, StorageConformanceProvider } from "./types.ts";
+
+export interface StorageConformanceRunner {
+	readonly describe: (name: string, suite: () => void) => unknown;
+	readonly expect: ExpectLike;
+	readonly it: (name: string, test: () => Promise<void>, timeoutMs?: number) => unknown;
+}
+
+/** Registers the runner-independent cases with a Vitest/Jest-compatible test runner. */
+export function registerStorageConformance(
+	runner: StorageConformanceRunner,
+	name: string,
+	withStorage: StorageConformanceProvider,
+): void {
+	const cases = createStorageConformance({
+		assertions: createExpectAssertions(runner.expect),
+		withStorage,
+	});
+	runner.describe(name, () => {
+		for (const testCase of cases) runner.it(testCase.name, testCase.run);
+	});
+}
+
+/** Registers the runner-independent `ExecutionEnv` cases with a Vitest/Jest-compatible test runner. */
+export function registerEnvConformance(
+	runner: StorageConformanceRunner,
+	name: string,
+	withEnv: EnvConformanceProvider,
+	options: Pick<EnvConformanceOptions, "shell" | "symlinks" | "listFiles"> = {},
+): void {
+	const cases = createEnvConformance({ ...options, assertions: createExpectAssertions(runner.expect), withEnv });
+	runner.describe(name, () => {
+		for (const testCase of cases) runner.it(testCase.name, testCase.run, testCase.timeoutMs);
+	});
+}

third_party/pi/packages/durable/src/testing/storage-benchmark.ts

@@ -0,0 +1,489 @@
+import type { JsonValue } from "@earendil-works/chord";
+import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
+import type {
+	ConversationId,
+	DocumentCreate,
+	DocumentId,
+	EntryId,
+	Seq,
+	Storage,
+	StorageWrite,
+	SubmissionId,
+	TaskId,
+	TaskRecord,
+} from "../types.ts";
+import { ROOT_CONVERSATION_ID } from "../types.ts";
+
+export type StorageBenchmarkScale = {
+	readonly name: string;
+	readonly entryCount: number;
+	readonly taskCount: number;
+	readonly documentCount: number;
+};
+
+export const STORAGE_MEMORY_SCALES: readonly StorageBenchmarkScale[] = [
+	{ name: "1k", entryCount: 1_000, taskCount: 200, documentCount: 200 },
+	{ name: "10k", entryCount: 10_000, taskCount: 2_000, documentCount: 2_000 },
+];
+
+export const TIMING_SCALE: StorageBenchmarkScale = {
+	name: "timing",
+	entryCount: 1_000,
+	taskCount: 300,
+	documentCount: 300,
+};
+
+const REPLAY_TAILS = [0, 16, 128, 1_024] as const;
+const HISTORY_SEGMENT_LENGTH = 128;
+const FORK_DEPTH = 8;
+const ENTRIES_PER_FORK = 32;
+const BATCH_SIZE = 100;
+
+export function storageBenchmarkPrimaryRecordCount(scale: StorageBenchmarkScale): number {
+	return (
+		1 +
+		scale.entryCount +
+		scale.taskCount +
+		scale.documentCount +
+		REPLAY_TAILS.length +
+		1 +
+		FORK_DEPTH * (1 + ENTRIES_PER_FORK)
+	);
+}
+
+type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
+
+export type StorageBenchmarkDataset = {
+	readonly firstEntryId: EntryId;
+	readonly filteredTaskCount: number;
+	readonly exactDocumentId: DocumentId;
+	readonly exactDocumentKey: string;
+	readonly replayDocumentIds: Readonly<Record<(typeof REPLAY_TAILS)[number], DocumentId>>;
+	readonly historicalDocumentId: DocumentId;
+	readonly ancientAt: Seq;
+	readonly recentAt: Seq;
+	readonly deepestConversationId: ConversationId;
+	readonly ancestorHeadEntryId: EntryId;
+};
+
+function task(id: TaskId<JsonValue>, index: number): StoredTask {
+	const statuses = ["pending", "running", "terminal"] as const;
+	const status = statuses[index % statuses.length];
+	const common = {
+		id,
+		conversationId: ROOT_CONVERSATION_ID,
+		kind: index % 4 === 0 ? "benchmark.filtered" : "benchmark.other",
+		version: 1,
+		input: { index },
+		background: index % 5 === 0,
+		abortRequested: index % 7 === 0,
+	};
+	if (status === "terminal") {
+		return { ...common, state: { status, outcome: { status: "completed", result: { index } } } };
+	}
+	return { ...common, state: { status, checkpoint: { index, payload: "x".repeat(64) } } };
+}
+
+/** Seed deterministic representative data through only the public `Storage` contract. */
+export async function seedStorageBenchmark(
+	storage: Storage,
+	scale: StorageBenchmarkScale = TIMING_SCALE,
+): Promise<StorageBenchmarkDataset> {
+	await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT);
+
+	let firstEntryId: EntryId | undefined;
+	for (let start = 0; start < scale.entryCount; start += BATCH_SIZE) {
+		const writes: StorageWrite[] = [];
+		for (let index = start; index < Math.min(start + BATCH_SIZE, scale.entryCount); index++) {
+			const id = await storage.mintId<EntryId>();
+			if (index === 0) firstEntryId = id;
+			writes.push({
+				type: "entry",
+				value: {
+					id,
+					conversationId: ROOT_CONVERSATION_ID,
+					kind: "benchmark.entry",
+					...(index === 0 ? { head: id } : {}),
+					data: { index, text: `entry-${index}-${"x".repeat(96)}` },
+				},
+			});
+		}
+		await storage.commit(writes, BACKGROUND_CONTEXT);
+	}
+
+	for (let start = 0; start < scale.taskCount; start += BATCH_SIZE) {
+		const writes: StorageWrite[] = [];
+		for (let index = start; index < Math.min(start + BATCH_SIZE, scale.taskCount); index++) {
+			writes.push({ type: "task", value: task(await storage.mintId<TaskId<JsonValue>>(), index) });
+		}
+		await storage.commit(writes, BACKGROUND_CONTEXT);
+	}
+
+	let exactDocumentId: DocumentId | undefined;
+	for (let start = 0; start < scale.documentCount; start += BATCH_SIZE) {
+		const writes: StorageWrite[] = [];
+		for (let index = start; index < Math.min(start + BATCH_SIZE, scale.documentCount); index++) {
+			const id = await storage.mintId<DocumentId>();
+			exactDocumentId = id;
+			writes.push({
+				type: "document.create",
+				record: { id, kind: "benchmark.family", key: `key-${index}`, scope: { kind: "session" } },
+				content: { kind: "base", version: 1, value: { index, text: "x".repeat(128) } },
+			});
+		}
+		await storage.commit(writes, BACKGROUND_CONTEXT);
+	}
+
+	const replayEntries = await Promise.all(
+		REPLAY_TAILS.map(async (tail) => {
+			const id = await storage.mintId<DocumentId>();
+			return { tail, id };
+		}),
+	);
+	await storage.commit(
+		replayEntries.map(({ tail, id }) => ({
+			type: "document.create" as const,
+			record: {
+				id,
+				kind: "benchmark.replay",
+				key: String(tail),
+				scope: { kind: "conversation" as const, conversationId: ROOT_CONVERSATION_ID },
+				history: "rewindable" as const,
+				fork: "asOf" as const,
+			},
+			content: { kind: "base" as const, version: 1, value: { count: 0, text: "x".repeat(64) } },
+		})),
+		BACKGROUND_CONTEXT,
+	);
+	for (let count = 1; count <= REPLAY_TAILS.at(-1)!; count++) {
+		await storage.commit(
+			replayEntries
+				.filter(({ tail }) => count <= tail)
+				.map(({ id }) => ({
+					type: "document.change" as const,
+					id,
+					content: { kind: "delta" as const, version: 1, ops: [["s", ["count"], count] as const] },
+				})),
+			BACKGROUND_CONTEXT,
+		);
+	}
+
+	const historicalDocumentId = await storage.mintId<DocumentId>();
+	const historicalRecord = {
+		id: historicalDocumentId,
+		kind: "benchmark.history",
+		scope: { kind: "conversation", conversationId: ROOT_CONVERSATION_ID },
+		history: "rewindable",
+		fork: "asOf",
+	} satisfies DocumentCreate;
+	await storage.commit(
+		[
+			{
+				type: "document.create",
+				record: historicalRecord,
+				content: { kind: "base", version: 1, value: { count: 0 } },
+			},
+		],
+		BACKGROUND_CONTEXT,
+	);
+	let ancientAt: Seq | undefined;
+	for (let count = 1; count <= HISTORY_SEGMENT_LENGTH; count++) {
+		ancientAt = await storage.commit(
+			[
+				{
+					type: "document.change",
+					id: historicalDocumentId,
+					content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] },
+				},
+			],
+			BACKGROUND_CONTEXT,
+		);
+	}
+	await storage.commit(
+		[
+			{
+				type: "document.change",
+				id: historicalDocumentId,
+				content: { kind: "base", version: 1, value: { count: HISTORY_SEGMENT_LENGTH } },
+			},
+		],
+		BACKGROUND_CONTEXT,
+	);
+	if (ancientAt === undefined) throw new Error("Benchmark history seed produced no commits");
+	let recentAt = ancientAt;
+	for (let count = HISTORY_SEGMENT_LENGTH + 1; count <= HISTORY_SEGMENT_LENGTH * 2; count++) {
+		recentAt = await storage.commit(
+			[
+				{
+					type: "document.change",
+					id: historicalDocumentId,
+					content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] },
+				},
+			],
+			BACKGROUND_CONTEXT,
+		);
+	}
+
+	if (firstEntryId === undefined) throw new Error("Benchmark scale must create entries");
+	let parentConversationId = ROOT_CONVERSATION_ID;
+	let parentAt = firstEntryId;
+	let deepestConversationId = ROOT_CONVERSATION_ID;
+	for (let depth = 0; depth < FORK_DEPTH; depth++) {
+		const conversationId = await storage.mintId<ConversationId>();
+		await storage.commit(
+			[
+				{
+					type: "conversation",
+					value: { id: conversationId, parent: { conversationId: parentConversationId, at: parentAt } },
+				},
+			],
+			BACKGROUND_CONTEXT,
+		);
+		const ids = await Promise.all(Array.from({ length: ENTRIES_PER_FORK }, () => storage.mintId<EntryId>()));
+		await storage.commit(
+			ids.map((id, index) => ({
+				type: "entry" as const,
+				value: {
+					id,
+					conversationId,
+					kind: "benchmark.fork",
+					data: { depth, index },
+				},
+			})),
+			BACKGROUND_CONTEXT,
+		);
+		parentConversationId = conversationId;
+		parentAt = ids.at(-1)!;
+		deepestConversationId = conversationId;
+	}
+
+	if (exactDocumentId === undefined) throw new Error("Benchmark scale must create documents");
+	return {
+		firstEntryId,
+		filteredTaskCount: Math.min(50, Math.ceil(scale.taskCount / 60)),
+		exactDocumentId,
+		exactDocumentKey: `key-${scale.documentCount - 1}`,
+		replayDocumentIds: Object.fromEntries(replayEntries.map(({ tail, id }) => [tail, id])) as Record<
+			(typeof REPLAY_TAILS)[number],
+			DocumentId
+		>,
+		historicalDocumentId,
+		ancientAt,
+		recentAt,
+		deepestConversationId,
+		ancestorHeadEntryId: firstEntryId,
+	};
+}
+
+export type StorageReadBenchmark = {
+	readonly name: string;
+	run(storage: Storage, dataset: StorageBenchmarkDataset): Promise<number>;
+	expected(dataset: StorageBenchmarkDataset): number;
+};
+
+export const STORAGE_READ_BENCHMARKS: readonly StorageReadBenchmark[] = [
+	{
+		name: "exact entry lookup",
+		async run(storage, dataset) {
+			return (await storage.entry(dataset.firstEntryId, BACKGROUND_CONTEXT))?.entry.id ?? -1;
+		},
+		expected: ({ firstEntryId }) => firstEntryId,
+	},
+	{
+		name: "entry page scan (100)",
+		async run(storage) {
+			return (
+				await storage.scanEntries({ conversationId: ROOT_CONVERSATION_ID }, 100, undefined, BACKGROUND_CONTEXT)
+			).items.length;
+		},
+		expected: () => 100,
+	},
+	{
+		name: "filtered task scan (50)",
+		async run(storage) {
+			return (
+				await storage.scanTasks(
+					{ kind: "benchmark.filtered", status: "pending", background: true },
+					50,
+					undefined,
+					BACKGROUND_CONTEXT,
+				)
+			).items.length;
+		},
+		expected: ({ filteredTaskCount }) => filteredTaskCount,
+	},
+	{
+		name: "exact document address among many",
+		async run(storage, dataset) {
+			return (
+				(
+					await storage.findDocument(
+						{ kind: "benchmark.family", key: dataset.exactDocumentKey, scope: { kind: "session" } },
+						"current",
+						BACKGROUND_CONTEXT,
+					)
+				)?.id ?? -1
+			);
+		},
+		expected: ({ exactDocumentId }) => exactDocumentId,
+	},
+	...REPLAY_TAILS.map(
+		(tail): StorageReadBenchmark => ({
+			name: `document replay tail (${tail})`,
+			async run(storage, dataset) {
+				return Number(
+					(await storage.document(dataset.replayDocumentIds[tail], "current", BACKGROUND_CONTEXT))?.value.count,
+				);
+			},
+			expected: () => tail,
+		}),
+	),
+	{
+		name: "ancient historical read before newer base",
+		async run(storage, dataset) {
+			return Number(
+				(await storage.document(dataset.historicalDocumentId, dataset.ancientAt, BACKGROUND_CONTEXT))?.value.count,
+			);
+		},
+		expected: () => HISTORY_SEGMENT_LENGTH,
+	},
+	{
+		name: "recent historical read after newer base",
+		async run(storage, dataset) {
+			return Number(
+				(await storage.document(dataset.historicalDocumentId, dataset.recentAt, BACKGROUND_CONTEXT))?.value.count,
+			);
+		},
+		expected: () => HISTORY_SEGMENT_LENGTH * 2,
+	},
+	{
+		name: "fork-depth history scan (100)",
+		async run(storage, dataset) {
+			return (
+				await storage.scanEntries(
+					{ conversationId: dataset.deepestConversationId },
+					100,
+					undefined,
+					BACKGROUND_CONTEXT,
+				)
+			).items.length;
+		},
+		expected: () => 100,
+	},
+	{
+		name: "fork-depth head lookup",
+		async run(storage, dataset) {
+			return (
+				(await storage.findLatestHeadMarker(dataset.deepestConversationId, undefined, BACKGROUND_CONTEXT))?.id ?? -1
+			);
+		},
+		expected: ({ ancestorHeadEntryId }) => ancestorHeadEntryId,
+	},
+];
+
+export type StorageWriteBenchmark = {
+	readonly name: string;
+	readonly expected: number;
+	run(storage: Storage): Promise<number>;
+};
+
+/** Seed the common state expected by every write benchmark sample. */
+export async function seedStorageWriteBenchmark(storage: Storage): Promise<void> {
+	await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT);
+	await storage.commit(
+		await Promise.all(
+			Array.from({ length: 100 }, async (_, index) => ({
+				type: "entry" as const,
+				value: {
+					id: await storage.mintId<EntryId>(),
+					conversationId: ROOT_CONVERSATION_ID,
+					kind: "benchmark.baseline",
+					data: { index },
+				},
+			})),
+		),
+		BACKGROUND_CONTEXT,
+	);
+}
+
+export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [
+	{
+		name: "commit one entry",
+		expected: 1,
+		async run(storage) {
+			const id = await storage.mintId<EntryId>();
+			await storage.commit(
+				[
+					{
+						type: "entry",
+						value: {
+							id,
+							conversationId: ROOT_CONVERSATION_ID,
+							kind: "benchmark.write",
+							data: { text: "x".repeat(128) },
+						},
+					},
+				],
+				BACKGROUND_CONTEXT,
+			);
+			return 1;
+		},
+	},
+	{
+		name: "commit 100 entries",
+		expected: 100,
+		async run(storage) {
+			const writes = await Promise.all(
+				Array.from(
+					{ length: 100 },
+					async (_, index): Promise<StorageWrite> => ({
+						type: "entry",
+						value: {
+							id: await storage.mintId<EntryId>(),
+							conversationId: ROOT_CONVERSATION_ID,
+							kind: "benchmark.write",
+							data: { index, text: "x".repeat(128) },
+						},
+					}),
+				),
+			);
+			await storage.commit(writes, BACKGROUND_CONTEXT);
+			return writes.length;
+		},
+	},
+	{
+		name: "commit mixed entry/task/submission/document",
+		expected: 4,
+		async run(storage) {
+			const entryId = await storage.mintId<EntryId>();
+			const taskId = await storage.mintId<TaskId<JsonValue>>();
+			const submissionId = await storage.mintId<SubmissionId>();
+			const documentId = await storage.mintId<DocumentId>();
+			const writes: readonly StorageWrite[] = [
+				{
+					type: "entry",
+					value: { id: entryId, conversationId: ROOT_CONVERSATION_ID, kind: "benchmark.mixed" },
+				},
+				{ type: "task", value: task(taskId, taskId) },
+				{
+					type: "submission",
+					value: {
+						id: submissionId,
+						conversationId: ROOT_CONVERSATION_ID,
+						requestId: `benchmark-${submissionId}`,
+						type: "write",
+						status: "done",
+						entry: entryId,
+					},
+				},
+				{
+					type: "document.create",
+					record: { id: documentId, kind: "benchmark.mixed", key: String(documentId), scope: { kind: "session" } },
+					content: { kind: "base", version: 1, value: { entryId, taskId } },
+				},
+			];
+			await storage.commit(writes, BACKGROUND_CONTEXT);
+			return writes.length;
+		},
+	},
+];

third_party/pi/packages/durable/src/testing/types.ts

@@ -0,0 +1,46 @@
+import type { ExecutionEnv } from "../env/index.ts";
+import type { Storage } from "../types.ts";
+
+export interface StorageConformanceAssertions {
+	ok(value: unknown, message?: string): void;
+	strictEqual(actual: unknown, expected: unknown): void;
+	deepEqual(actual: unknown, expected: unknown): void;
+	partialDeepEqual(actual: unknown, expected: unknown): void;
+	greaterThan(actual: number, expected: number): void;
+	rejects(operation: Promise<unknown>, messageIncludes: string): Promise<void>;
+}
+
+export type StorageConformanceProvider = (use: (storage: Storage) => Promise<void>) => Promise<void>;
+
+export interface StorageConformanceOptions {
+	readonly assertions: StorageConformanceAssertions;
+	readonly withStorage: StorageConformanceProvider;
+}
+
+export interface StorageConformanceCase {
+	readonly name: string;
+	run(): Promise<void>;
+}
+
+export type EnvConformanceAssertions = StorageConformanceAssertions;
+
+/** Calls `use` once with an environment whose `cwd` is a fresh, empty, writable directory, then cleans up. */
+export type EnvConformanceProvider = (use: (env: ExecutionEnv) => Promise<void>) => Promise<void>;
+
+export interface EnvConformanceOptions {
+	readonly assertions: EnvConformanceAssertions;
+	readonly withEnv: EnvConformanceProvider;
+	/** Program and flag that run a POSIX shell script from the next argument; default `["sh", "-c"]`. */
+	readonly shell?: readonly string[];
+	/** Whether the shell's `ln -s` creates symbolic links; default true. */
+	readonly symlinks?: boolean;
+	/** Whether the environment implements the optional `listFiles`, which the suite then checks; default false. */
+	readonly listFiles?: boolean;
+}
+
+export interface EnvConformanceCase {
+	readonly name: string;
+	/** Cases that wait for an environment's watch latency need longer than a test runner's default timeout. */
+	readonly timeoutMs?: number;
+	run(): Promise<void>;
+}

third_party/pi/packages/durable/src/tool-search/index.ts

@@ -0,0 +1,7 @@
+export {
+	createToolSearchExtension,
+	createToolSearchTool,
+	DEFAULT_TOOL_SEARCH_LIMIT,
+	TOOL_SEARCH_TOOL_NAME,
+	type ToolSearchToolDetails,
+} from "./tool-search.ts";

third_party/pi/packages/durable/src/tool-search/tool-search.ts

@@ -0,0 +1,92 @@
+/**
+ * The `tool_search` tool, a copy of the coding agent's: the model searches the deferred tools it is not offered yet,
+ * and the matches are loaded with `control.addTools`, so the next request offers them. Loading writes the
+ * conversation's `modelTools` (its `add`), so loaded tools stay offered on that branch, also after a compaction.
+ */
+import { Type } from "@earendil-works/pi-ai";
+import { defineExtension, defineTool } from "../harness/define.ts";
+import type { ToolRegistration } from "../harness/types.ts";
+import { rankDocuments } from "../search.ts";
+
+export const TOOL_SEARCH_TOOL_NAME = "tool_search";
+export const DEFAULT_TOOL_SEARCH_LIMIT = 8;
+
+/** Fixed, so it does not change while deferred tools are registered, for example as MCP servers connect. */
+const TOOL_SEARCH_DESCRIPTION = `# Tool discovery\n\nSearches over deferred tool metadata with BM25 and exposes matching tools for the next model call.\n\nSome of the tools, such as tools of MCP servers, may not have been provided to you upfront, and you should use this tool (\`${TOOL_SEARCH_TOOL_NAME}\`) to search for the required tools. For MCP tool discovery, always use \`${TOOL_SEARCH_TOOL_NAME}\`.`;
+
+const toolSearchSchema = Type.Object({
+	query: Type.String({ description: "Search query for deferred tools." }),
+	limit: Type.Optional(
+		Type.Number({ description: `Maximum number of tools to return. Defaults to ${DEFAULT_TOOL_SEARCH_LIMIT}.` }),
+	),
+});
+
+/** `tool_search` result details: the tools it loaded. */
+export type ToolSearchToolDetails = { loaded: string[] };
+
+function isObject(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/** Schema descriptions and property names, recursively. */
+function schemaText(schema: unknown, parts: string[]): void {
+	if (!isObject(schema)) return;
+	if (typeof schema.description === "string") parts.push(schema.description);
+	if (isObject(schema.properties)) {
+		for (const [name, property] of Object.entries(schema.properties)) {
+			parts.push(name);
+			schemaText(property, parts);
+		}
+	}
+	schemaText(schema.items, parts);
+	for (const key of ["anyOf", "oneOf", "allOf"]) {
+		const variants = schema[key];
+		if (Array.isArray(variants)) for (const variant of variants) schemaText(variant, parts);
+	}
+}
+
+/** Search text of a tool: its name, the name with `_` as spaces, its description, and its parameters' text. */
+function searchText(tool: ToolRegistration): string {
+	const parts = [tool.name, tool.name.replaceAll("_", " "), tool.description];
+	schemaText(tool.parameters, parts);
+	return parts.filter((part) => part.trim() !== "").join(" ");
+}
+
+export function createToolSearchTool() {
+	return defineTool({
+		name: TOOL_SEARCH_TOOL_NAME,
+		description: TOOL_SEARCH_DESCRIPTION,
+		parameters: toolSearchSchema,
+		// Searching changes what the model is offered; scripts have their own `searchTools()`.
+		callers: ["model"],
+		replay: "safe",
+		async execute({ query, limit }, api, context) {
+			if (query.trim() === "") throw new Error("query must not be empty");
+			const max = limit ?? DEFAULT_TOOL_SEARCH_LIMIT;
+			if (!Number.isInteger(max) || max <= 0) throw new Error("limit must be a positive integer");
+			const candidates = (await api.agent(context)).deferred;
+			const documents = candidates.map((tool) => ({ name: tool.name, text: searchText(tool) }));
+			const loaded = rankDocuments(query, documents, max);
+			const text =
+				loaded.length === 0
+					? "No matching tools found."
+					: `Loaded ${loaded.length} tool${loaded.length === 1 ? "" : "s"}. They are available from your next call:\n${loaded
+							.map((name) => {
+								const description = candidates.find((tool) => tool.name === name)?.description ?? "";
+								return `- ${name}: ${description.trim().split(/\r?\n/)[0]}`;
+							})
+							.join("\n")}`;
+			const details: ToolSearchToolDetails = { loaded };
+			return {
+				output: [{ type: "text", text }],
+				details,
+				...(loaded.length === 0 ? {} : { control: { addTools: loaded } }),
+			};
+		},
+	});
+}
+
+/** The `tool_search` extension: install it and select it where the model should find deferred tools. */
+export function createToolSearchExtension() {
+	return defineExtension({ name: "tool-search", tools: [createToolSearchTool()] });
+}

third_party/pi/packages/durable/src/tools/bash.ts

@@ -0,0 +1,184 @@
+import type { Context } from "@earendil-works/chord";
+import { type Static, Type } from "typebox";
+import { defineTool } from "../harness/define.ts";
+import type { ToolExecutionApi, ToolExecutionResult, ToolRegistration } from "../harness/types.ts";
+import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "../truncate.ts";
+import { requireEnv } from "./env.ts";
+
+const MAX_TIMEOUT_SECONDS = 2_147_483_647 / 1000;
+
+const bashSchema = Type.Object({
+	command: Type.String({ description: "Bash command to execute" }),
+	timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })),
+});
+
+const powershellSchema = Type.Object({
+	command: Type.String({ description: "PowerShell command to execute" }),
+	timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })),
+});
+
+/** What a program that calls `bash` or `powershell` receives. */
+const shellStructuredOutputSchema = Type.Object({
+	output: Type.String({ description: "Combined stdout and stderr: the retained tail, as the model sees it" }),
+	truncated: Type.Boolean({ description: "Whether earlier output was dropped" }),
+	fullOutputPath: Type.Optional(Type.String({ description: "File with the complete output, when it was spilled" })),
+	exitCode: Type.Number(),
+});
+
+export type BashToolInput = Static<typeof bashSchema>;
+export type PowerShellToolInput = Static<typeof powershellSchema>;
+export type ShellStructuredOutput = Static<typeof shellStructuredOutputSchema>;
+
+/** A command about to run, which `prepare` may change: the script, its working directory and environment. */
+export interface BashExecution {
+	command: string;
+	cwd: string;
+	env: Record<string, string>;
+	inheritEnv: boolean;
+}
+
+export type BashPrepare = (execution: BashExecution, api: ToolExecutionApi, context: Context) => void | Promise<void>;
+
+export interface BashToolOptions {
+	commandPrefix?: string;
+	prepare?: BashPrepare;
+}
+
+export interface PowerShellToolOptions {
+	/** Lines run before each command. */
+	commandPrefix?: string;
+	prepare?: BashPrepare;
+	/** PowerShell programs to try in order; default `pwsh`, then `powershell`. A program that cannot start is skipped. */
+	programs?: readonly string[];
+}
+
+function validateTimeout(timeout: number | undefined): void {
+	if (timeout === undefined) return;
+	if (!Number.isFinite(timeout) || timeout <= 0) {
+		throw new Error("Invalid timeout: must be a finite number of seconds");
+	}
+	if (timeout > MAX_TIMEOUT_SECONDS) {
+		throw new Error(`Invalid timeout: maximum is ${MAX_TIMEOUT_SECONDS} seconds`);
+	}
+}
+
+/** The execution of one call: the command with its prefix, in the environment's working directory, then `prepare`. */
+async function prepareExecution(
+	command: string,
+	options: { commandPrefix?: string; prepare?: BashPrepare } | undefined,
+	api: ToolExecutionApi,
+	context: Context,
+): Promise<BashExecution> {
+	const execution: BashExecution = {
+		command: options?.commandPrefix ? `${options.commandPrefix}\n${command}` : command,
+		cwd: requireEnv(api).cwd,
+		env: {},
+		inheritEnv: true,
+	};
+	await options?.prepare?.(execution, api, context);
+	return execution;
+}
+
+/**
+ * Run each command form in turn until one starts, streaming output to `api.output()` within the retained window, and
+ * turn the result into the tool's result: a spill diagnostic, an error result for a nonzero exit, and a thrown error for
+ * a failure such as a timeout. The structured output repeats the retained output with the exit code.
+ */
+async function runCommand(
+	commands: readonly (string | readonly string[])[],
+	execution: BashExecution,
+	timeout: number | undefined,
+	api: ToolExecutionApi,
+	context: Context,
+): Promise<ToolExecutionResult> {
+	const env = requireEnv(api);
+	let result: Awaited<ReturnType<typeof env.exec>> | undefined;
+	for (const command of commands) {
+		result = await env.exec(
+			command,
+			{
+				cwd: execution.cwd,
+				env: execution.env,
+				inheritEnv: execution.inheritEnv,
+				...(timeout === undefined ? {} : { timeout }),
+				onOutput: (text, _context, info) => api.output(text, info.skipped),
+				spill: { afterBytes: DEFAULT_MAX_BYTES, afterLines: DEFAULT_MAX_LINES },
+				// An environment may then omit output outside the retained tail and report the omission.
+				...(api.outputWindow === undefined ? {} : { window: api.outputWindow }),
+			},
+			context,
+		);
+		// A program that could not start produced no output; the next form may.
+		if (result.ok || result.error.code !== "spawn_error") break;
+	}
+	if (result === undefined) throw new Error("No command to run");
+	const spillPath = result.ok ? result.value.spillPath : result.error.spillPath;
+	if (spillPath !== undefined) {
+		api.diagnostic({ severity: "info", code: "full_output", message: `Full output: ${spillPath}` });
+	}
+	if (!result.ok) {
+		if (result.error.code === "aborted" && context.abortSignal?.aborted) throw result.error;
+		if (result.error.code === "timeout") throw new Error(`Command timed out after ${timeout} seconds`);
+		if (result.error.code === "aborted") throw new Error("Command aborted");
+		throw result.error;
+	}
+	const exitCode = result.value.exitCode;
+	const retained = api.retainedOutput();
+	const structuredOutput: ShellStructuredOutput = {
+		output: retained.text,
+		truncated: retained.truncated,
+		...(spillPath === undefined ? {} : { fullOutputPath: spillPath }),
+		exitCode,
+	};
+	if (exitCode === 0) return { structuredOutput };
+	const exited = { severity: "error", code: "exit_code", message: `Command exited with code ${exitCode}` } as const;
+	return { structuredOutput, isError: true, diagnostics: [exited] };
+}
+
+/**
+ * Runs a command through the environment's shell. Its output streams to `api.output()`, where the Harness keeps the
+ * tail within the default limits; the result content is that retained output. The retained window goes to the
+ * environment, which may omit output outside it and report how much it omitted, so dropped counts stay exact. Output
+ * beyond the limits is spilled to a file whose path is reported as a diagnostic. A nonzero exit is an error result with
+ * the exit code; a timeout throws, which makes an error result that still carries the output and diagnostics.
+ */
+export function createBashTool(options?: BashToolOptions): ToolRegistration<typeof bashSchema> {
+	return defineTool({
+		name: "bash",
+		description: `Execute a bash command in the current working directory. Returns combined stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`,
+		parameters: bashSchema,
+		structuredOutputSchema: shellStructuredOutputSchema,
+		outputLimits: { retain: "tail" },
+		async execute(args, api, context) {
+			validateTimeout(args.timeout);
+			const execution = await prepareExecution(args.command, options, api, context);
+			return runCommand([execution.command], execution, args.timeout, api, context);
+		},
+	});
+}
+
+/** Output in UTF-8 whatever the console's code page, as the coding agent's `powershell` tool does. */
+const UTF8_OUTPUT = "try { [Console]::OutputEncoding=[System.Text.Encoding]::UTF8 } catch {}";
+const POWERSHELL_ARGS = ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command"] as const;
+
+/**
+ * Runs a PowerShell command, like `bash` but through PowerShell instead of the environment's shell: `pwsh` (PowerShell
+ * 7), else Windows PowerShell, started directly with the command as an argument, so no other shell parses it.
+ */
+export function createPowerShellTool(options?: PowerShellToolOptions): ToolRegistration<typeof powershellSchema> {
+	const programs = options?.programs ?? ["pwsh", "powershell"];
+	return defineTool({
+		name: "powershell",
+		description: `Execute a PowerShell command in the current working directory. Returns combined stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`,
+		parameters: powershellSchema,
+		structuredOutputSchema: shellStructuredOutputSchema,
+		outputLimits: { retain: "tail" },
+		async execute(args, api, context) {
+			validateTimeout(args.timeout);
+			const execution = await prepareExecution(args.command, options, api, context);
+			const script = `${UTF8_OUTPUT}\n${execution.command}`;
+			const commands = programs.map((program) => [program, ...POWERSHELL_ARGS, script]);
+			return runCommand(commands, execution, args.timeout, api, context);
+		},
+	});
+}

third_party/pi/packages/durable/src/tools/edit-diff.ts

@@ -0,0 +1,500 @@
+/**
+ * Shared diff computation utilities for the edit and similar tools.
+ */
+
+import * as Diff from "diff";
+
+export function detectLineEnding(content: string): "\r\n" | "\n" {
+	const crlfIdx = content.indexOf("\r\n");
+	const lfIdx = content.indexOf("\n");
+	if (lfIdx === -1) return "\n";
+	if (crlfIdx === -1) return "\n";
+	return crlfIdx < lfIdx ? "\r\n" : "\n";
+}
+
+export function normalizeToLF(text: string): string {
+	return text.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
+}
+
+export function restoreLineEndings(text: string, ending: "\r\n" | "\n"): string {
+	return ending === "\r\n" ? text.replace(/\n/g, "\r\n") : text;
+}
+
+/**
+ * Normalize text for fuzzy matching. Applies progressive transformations:
+ * - Strip trailing whitespace from each line
+ * - Normalize smart quotes to ASCII equivalents
+ * - Normalize Unicode dashes/hyphens to ASCII hyphen
+ * - Normalize special Unicode spaces to regular space
+ */
+export function normalizeForFuzzyMatch(text: string): string {
+	return (
+		text
+			.normalize("NFKC")
+			// Strip trailing whitespace per line
+			.split("\n")
+			.map((line) => line.trimEnd())
+			.join("\n")
+			// Smart single quotes → '
+			.replace(/[\u2018\u2019\u201A\u201B]/g, "'")
+			// Smart double quotes → "
+			.replace(/[\u201C\u201D\u201E\u201F]/g, '"')
+			// Various dashes/hyphens → -
+			// U+2010 hyphen, U+2011 non-breaking hyphen, U+2012 figure dash,
+			// U+2013 en-dash, U+2014 em-dash, U+2015 horizontal bar, U+2212 minus
+			.replace(/[\u2010\u2011\u2012\u2013\u2014\u2015\u2212]/g, "-")
+			// Special spaces → regular space
+			// U+00A0 NBSP, U+2002-U+200A various spaces, U+202F narrow NBSP,
+			// U+205F medium math space, U+3000 ideographic space
+			.replace(/[\u00A0\u2002-\u200A\u202F\u205F\u3000]/g, " ")
+	);
+}
+
+function splitLinesWithEndings(content: string): string[] {
+	return content.match(/[^\n]*\n|[^\n]+/g) ?? [];
+}
+
+interface LineSpan {
+	start: number;
+	end: number;
+}
+
+interface MatchedEdit {
+	editIndex: number;
+	matchIndex: number;
+	matchLength: number;
+	newText: string;
+}
+
+type TextReplacement = Pick<MatchedEdit, "matchIndex" | "matchLength" | "newText">;
+
+function getLineSpans(content: string): LineSpan[] {
+	let offset = 0;
+	return splitLinesWithEndings(content).map((line) => {
+		const span = { start: offset, end: offset + line.length };
+		offset = span.end;
+		return span;
+	});
+}
+
+function getReplacementLineRange(lines: LineSpan[], replacement: TextReplacement) {
+	const replacementStart = replacement.matchIndex;
+	const replacementEnd = replacement.matchIndex + replacement.matchLength;
+
+	let startLine = -1;
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i];
+		if (replacementStart >= line.start && replacementStart < line.end) {
+			startLine = i;
+			break;
+		}
+	}
+	if (startLine === -1) {
+		throw new Error("Replacement range is outside the base content.");
+	}
+
+	let endLine = startLine;
+	while (endLine < lines.length && lines[endLine].end < replacementEnd) {
+		endLine++;
+	}
+	if (endLine >= lines.length) {
+		throw new Error("Replacement range is outside the base content.");
+	}
+
+	return { startLine, endLine: endLine + 1 };
+}
+
+function applyReplacements(content: string, replacements: TextReplacement[], offset = 0): string {
+	let result = content;
+	for (let i = replacements.length - 1; i >= 0; i--) {
+		const replacement = replacements[i];
+		const matchIndex = replacement.matchIndex - offset;
+		result =
+			result.substring(0, matchIndex) + replacement.newText + result.substring(matchIndex + replacement.matchLength);
+	}
+	return result;
+}
+
+/**
+ * Apply replacements matched against `baseContent` to `originalContent` while
+ * preserving unchanged line blocks from the original.
+ *
+ * This is useful when `baseContent` is a normalized view of the original. Each
+ * replacement is widened to the lines it actually touches, those touched lines
+ * are rewritten from the normalized base, and all other lines are copied back
+ * from `originalContent`. The actual replacement ranges drive preservation so
+ * duplicate normalized lines cannot be aligned to the wrong occurrence.
+ */
+export function applyReplacementsPreservingUnchangedLines(
+	originalContent: string,
+	baseContent: string,
+	replacements: TextReplacement[],
+): string {
+	const originalLines = splitLinesWithEndings(originalContent);
+	const baseLines = getLineSpans(baseContent);
+	if (originalLines.length !== baseLines.length) {
+		throw new Error("Cannot preserve unchanged lines because the base content has a different line count.");
+	}
+
+	const groups: Array<{ startLine: number; endLine: number; replacements: TextReplacement[] }> = [];
+	const sortedReplacements = [...replacements].sort((a, b) => a.matchIndex - b.matchIndex);
+	for (const replacement of sortedReplacements) {
+		const range = getReplacementLineRange(baseLines, replacement);
+		const current = groups[groups.length - 1];
+		if (current && range.startLine < current.endLine) {
+			current.endLine = Math.max(current.endLine, range.endLine);
+			current.replacements.push(replacement);
+			continue;
+		}
+		groups.push({ ...range, replacements: [replacement] });
+	}
+
+	let originalLineIndex = 0;
+	let result = "";
+	for (const group of groups) {
+		result += originalLines.slice(originalLineIndex, group.startLine).join("");
+
+		const groupStartOffset = baseLines[group.startLine].start;
+		const groupEndOffset = baseLines[group.endLine - 1].end;
+		result += applyReplacements(
+			baseContent.slice(groupStartOffset, groupEndOffset),
+			group.replacements,
+			groupStartOffset,
+		);
+		originalLineIndex = group.endLine;
+	}
+	result += originalLines.slice(originalLineIndex).join("");
+
+	return result;
+}
+
+export interface FuzzyMatchResult {
+	/** Whether a match was found */
+	found: boolean;
+	/** The index where the match starts (in the content that should be used for replacement) */
+	index: number;
+	/** Length of the matched text */
+	matchLength: number;
+	/** Whether fuzzy matching was used (false = exact match) */
+	usedFuzzyMatch: boolean;
+	/**
+	 * The content to use for replacement operations.
+	 * When exact match: original content. When fuzzy match: normalized content.
+	 */
+	contentForReplacement: string;
+}
+
+export interface Edit {
+	oldText: string;
+	newText: string;
+}
+
+export interface AppliedEditsResult {
+	baseContent: string;
+	newContent: string;
+}
+
+/**
+ * Find oldText in content, trying exact match first, then fuzzy match.
+ * When fuzzy matching is used, the returned contentForReplacement is the
+ * fuzzy-normalized version of the content (trailing whitespace stripped,
+ * Unicode quotes/dashes normalized to ASCII).
+ */
+export function fuzzyFindText(content: string, oldText: string): FuzzyMatchResult {
+	// Try exact match first
+	const exactIndex = content.indexOf(oldText);
+	if (exactIndex !== -1) {
+		return {
+			found: true,
+			index: exactIndex,
+			matchLength: oldText.length,
+			usedFuzzyMatch: false,
+			contentForReplacement: content,
+		};
+	}
+
+	// Try fuzzy match - work entirely in normalized space
+	const fuzzyContent = normalizeForFuzzyMatch(content);
+	const fuzzyOldText = normalizeForFuzzyMatch(oldText);
+	const fuzzyIndex = fuzzyContent.indexOf(fuzzyOldText);
+
+	if (fuzzyIndex === -1) {
+		return {
+			found: false,
+			index: -1,
+			matchLength: 0,
+			usedFuzzyMatch: false,
+			contentForReplacement: content,
+		};
+	}
+
+	// When fuzzy matching, return offsets in normalized space. Callers can use
+	// the normalized content to compute replacements, then decide how much of
+	// that normalized output should be written back.
+	return {
+		found: true,
+		index: fuzzyIndex,
+		matchLength: fuzzyOldText.length,
+		usedFuzzyMatch: true,
+		contentForReplacement: fuzzyContent,
+	};
+}
+
+/** Strip UTF-8 BOM if present, return both the BOM (if any) and the text without it */
+export function stripBom(content: string): { bom: string; text: string } {
+	return content.startsWith("\uFEFF") ? { bom: "\uFEFF", text: content.slice(1) } : { bom: "", text: content };
+}
+
+function countOccurrences(content: string, oldText: string): number {
+	const fuzzyContent = normalizeForFuzzyMatch(content);
+	const fuzzyOldText = normalizeForFuzzyMatch(oldText);
+	return fuzzyContent.split(fuzzyOldText).length - 1;
+}
+
+function getNotFoundError(path: string, editIndex: number, totalEdits: number): Error {
+	if (totalEdits === 1) {
+		return new Error(
+			`Could not find the exact text in ${path}. The old text must match exactly including all whitespace and newlines.`,
+		);
+	}
+	return new Error(
+		`Could not find edits[${editIndex}] in ${path}. The oldText must match exactly including all whitespace and newlines.`,
+	);
+}
+
+function getDuplicateError(path: string, editIndex: number, totalEdits: number, occurrences: number): Error {
+	if (totalEdits === 1) {
+		return new Error(
+			`Found ${occurrences} occurrences of the text in ${path}. The text must be unique. Please provide more context to make it unique.`,
+		);
+	}
+	return new Error(
+		`Found ${occurrences} occurrences of edits[${editIndex}] in ${path}. Each oldText must be unique. Please provide more context to make it unique.`,
+	);
+}
+
+function getEmptyOldTextError(path: string, editIndex: number, totalEdits: number): Error {
+	if (totalEdits === 1) {
+		return new Error(`oldText must not be empty in ${path}.`);
+	}
+	return new Error(`edits[${editIndex}].oldText must not be empty in ${path}.`);
+}
+
+function getNoChangeError(path: string, totalEdits: number): Error {
+	if (totalEdits === 1) {
+		return new Error(
+			`No changes made to ${path}. The replacement produced identical content. This might indicate an issue with special characters or the text not existing as expected.`,
+		);
+	}
+	return new Error(`No changes made to ${path}. The replacements produced identical content.`);
+}
+
+/**
+ * Apply one or more exact-text replacements to LF-normalized content.
+ *
+ * All edits are matched against the same original content. Replacements are
+ * then applied in reverse order so offsets remain stable. If any edit needs
+ * fuzzy matching, the operation runs in fuzzy-normalized content space and then
+ * overlays those line-level changes onto the original content so unchanged line
+ * blocks keep their original bytes.
+ */
+export function applyEditsToNormalizedContent(
+	normalizedContent: string,
+	edits: Edit[],
+	path: string,
+): AppliedEditsResult {
+	const normalizedEdits = edits.map((edit) => ({
+		oldText: normalizeToLF(edit.oldText),
+		newText: normalizeToLF(edit.newText),
+	}));
+
+	for (let i = 0; i < normalizedEdits.length; i++) {
+		if (normalizedEdits[i].oldText.length === 0) {
+			throw getEmptyOldTextError(path, i, normalizedEdits.length);
+		}
+	}
+
+	const initialMatches = normalizedEdits.map((edit) => fuzzyFindText(normalizedContent, edit.oldText));
+	const usedFuzzyMatch = initialMatches.some((match) => match.usedFuzzyMatch);
+	const replacementBaseContent = usedFuzzyMatch ? normalizeForFuzzyMatch(normalizedContent) : normalizedContent;
+
+	const matchedEdits: MatchedEdit[] = [];
+	for (let i = 0; i < normalizedEdits.length; i++) {
+		const edit = normalizedEdits[i];
+		const matchResult = fuzzyFindText(replacementBaseContent, edit.oldText);
+		if (!matchResult.found) {
+			throw getNotFoundError(path, i, normalizedEdits.length);
+		}
+
+		const occurrences = countOccurrences(replacementBaseContent, edit.oldText);
+		if (occurrences > 1) {
+			throw getDuplicateError(path, i, normalizedEdits.length, occurrences);
+		}
+
+		matchedEdits.push({
+			editIndex: i,
+			matchIndex: matchResult.index,
+			matchLength: matchResult.matchLength,
+			newText: edit.newText,
+		});
+	}
+
+	matchedEdits.sort((a, b) => a.matchIndex - b.matchIndex);
+	for (let i = 1; i < matchedEdits.length; i++) {
+		const previous = matchedEdits[i - 1];
+		const current = matchedEdits[i];
+		if (previous.matchIndex + previous.matchLength > current.matchIndex) {
+			throw new Error(
+				`edits[${previous.editIndex}] and edits[${current.editIndex}] overlap in ${path}. Merge them into one edit or target disjoint regions.`,
+			);
+		}
+	}
+
+	const baseContent = normalizedContent;
+	const newContent = usedFuzzyMatch
+		? applyReplacementsPreservingUnchangedLines(normalizedContent, replacementBaseContent, matchedEdits)
+		: applyReplacements(replacementBaseContent, matchedEdits);
+
+	if (baseContent === newContent) {
+		throw getNoChangeError(path, normalizedEdits.length);
+	}
+
+	return { baseContent, newContent };
+}
+
+/** Generate a standard unified patch. */
+export function generateUnifiedPatch(path: string, oldContent: string, newContent: string, contextLines = 4): string {
+	return Diff.createTwoFilesPatch(path, path, oldContent, newContent, undefined, undefined, {
+		context: contextLines,
+		headerOptions: Diff.FILE_HEADERS_ONLY,
+	});
+}
+
+/**
+ * Generate a display-oriented diff string with line numbers and context.
+ * Returns both the diff string and the first changed line number (in the new file).
+ */
+export function generateDiffString(
+	oldContent: string,
+	newContent: string,
+	contextLines = 4,
+): { diff: string; firstChangedLine: number | undefined } {
+	const parts = Diff.diffLines(oldContent, newContent);
+	const output: string[] = [];
+
+	const oldLines = oldContent.split("\n");
+	const newLines = newContent.split("\n");
+	const maxLineNum = Math.max(oldLines.length, newLines.length);
+	const lineNumWidth = String(maxLineNum).length;
+
+	let oldLineNum = 1;
+	let newLineNum = 1;
+	let lastWasChange = false;
+	let firstChangedLine: number | undefined;
+
+	for (let i = 0; i < parts.length; i++) {
+		const part = parts[i];
+		const raw = part.value.split("\n");
+		if (raw[raw.length - 1] === "") {
+			raw.pop();
+		}
+
+		if (part.added || part.removed) {
+			// Capture the first changed line (in the new file)
+			if (firstChangedLine === undefined) {
+				firstChangedLine = newLineNum;
+			}
+
+			// Show the change
+			for (const line of raw) {
+				if (part.added) {
+					const lineNum = String(newLineNum).padStart(lineNumWidth, " ");
+					output.push(`+${lineNum} ${line}`);
+					newLineNum++;
+				} else {
+					// removed
+					const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
+					output.push(`-${lineNum} ${line}`);
+					oldLineNum++;
+				}
+			}
+			lastWasChange = true;
+		} else {
+			// Context lines - only show a few before/after changes
+			const nextPartIsChange = i < parts.length - 1 && (parts[i + 1].added || parts[i + 1].removed);
+			const hasLeadingChange = lastWasChange;
+			const hasTrailingChange = nextPartIsChange;
+
+			if (hasLeadingChange && hasTrailingChange) {
+				if (raw.length <= contextLines * 2) {
+					for (const line of raw) {
+						const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
+						output.push(` ${lineNum} ${line}`);
+						oldLineNum++;
+						newLineNum++;
+					}
+				} else {
+					const leadingLines = raw.slice(0, contextLines);
+					const trailingLines = raw.slice(raw.length - contextLines);
+					const skippedLines = raw.length - leadingLines.length - trailingLines.length;
+
+					for (const line of leadingLines) {
+						const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
+						output.push(` ${lineNum} ${line}`);
+						oldLineNum++;
+						newLineNum++;
+					}
+
+					output.push(` ${"".padStart(lineNumWidth, " ")} ...`);
+					oldLineNum += skippedLines;
+					newLineNum += skippedLines;
+
+					for (const line of trailingLines) {
+						const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
+						output.push(` ${lineNum} ${line}`);
+						oldLineNum++;
+						newLineNum++;
+					}
+				}
+			} else if (hasLeadingChange) {
+				const shownLines = raw.slice(0, contextLines);
+				const skippedLines = raw.length - shownLines.length;
+
+				for (const line of shownLines) {
+					const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
+					output.push(` ${lineNum} ${line}`);
+					oldLineNum++;
+					newLineNum++;
+				}
+
+				if (skippedLines > 0) {
+					output.push(` ${"".padStart(lineNumWidth, " ")} ...`);
+					oldLineNum += skippedLines;
+					newLineNum += skippedLines;
+				}
+			} else if (hasTrailingChange) {
+				const skippedLines = Math.max(0, raw.length - contextLines);
+				if (skippedLines > 0) {
+					output.push(` ${"".padStart(lineNumWidth, " ")} ...`);
+					oldLineNum += skippedLines;
+					newLineNum += skippedLines;
+				}
+
+				for (const line of raw.slice(skippedLines)) {
+					const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
+					output.push(` ${lineNum} ${line}`);
+					oldLineNum++;
+					newLineNum++;
+				}
+			} else {
+				// Skip these context lines entirely
+				oldLineNum += raw.length;
+				newLineNum += raw.length;
+			}
+
+			lastWasChange = false;
+		}
+	}
+
+	return { diff: output.join("\n"), firstChangedLine };
+}

third_party/pi/packages/durable/src/tools/edit.ts

@@ -0,0 +1,144 @@
+import { type Static, Type } from "typebox";
+import type { FileError } from "../env/index.ts";
+import { defineTool } from "../harness/define.ts";
+import type { ToolRegistration } from "../harness/types.ts";
+import {
+	applyEditsToNormalizedContent,
+	detectLineEnding,
+	type Edit,
+	generateDiffString,
+	generateUnifiedPatch,
+	normalizeToLF,
+	restoreLineEndings,
+	stripBom,
+} from "./edit-diff.ts";
+import { requireEnv } from "./env.ts";
+import { withFileMutationQueue } from "./file-mutation-queue.ts";
+import { resolveToolPath } from "./path-utils.ts";
+
+const replaceEditSchema = Type.Object({
+	oldText: Type.String({
+		description:
+			"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.",
+	}),
+	newText: Type.String({ description: "Replacement text for this targeted edit." }),
+});
+
+const editSchema = Type.Object({
+	path: Type.String({ description: "Path to the file to edit (relative or absolute)" }),
+	edits: Type.Array(replaceEditSchema, {
+		description:
+			"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.",
+	}),
+});
+
+export type EditToolInput = Static<typeof editSchema>;
+type LegacyEditToolInput = EditToolInput & { oldText?: unknown; newText?: unknown };
+type SingleEditInput = { oldText: string; newText: string };
+
+function isSingleEditInput(value: unknown): value is SingleEditInput {
+	if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+	const edit = value as Record<string, unknown>;
+	return typeof edit.oldText === "string" && typeof edit.newText === "string";
+}
+
+/**
+ * Repair shapes models commonly send: `edits` as a JSON string or as a single edit object, and a top-level
+ * `oldText`/`newText` pair. Works on a copy; the call's arguments stay unchanged.
+ */
+function prepareEditArguments(input: unknown): EditToolInput {
+	if (!input || typeof input !== "object" || Array.isArray(input)) return input as EditToolInput;
+	const args: Record<string, unknown> = { ...input };
+	if (typeof args.edits === "string") {
+		try {
+			const parsed: unknown = JSON.parse(args.edits);
+			if (Array.isArray(parsed)) {
+				args.edits = parsed;
+			} else if (isSingleEditInput(parsed)) {
+				args.edits = [parsed];
+			}
+		} catch {}
+	} else if (isSingleEditInput(args.edits)) {
+		args.edits = [args.edits];
+	}
+
+	const legacy = args as LegacyEditToolInput;
+	if (typeof legacy.oldText !== "string" || typeof legacy.newText !== "string") return legacy;
+	const edits = Array.isArray(legacy.edits) ? [...legacy.edits] : [];
+	edits.push({ oldText: legacy.oldText, newText: legacy.newText });
+	const { oldText: _oldText, newText: _newText, ...rest } = legacy;
+	return { ...rest, edits };
+}
+
+export type EditToolDetails = {
+	diff: string;
+	patch: string;
+	firstChangedLine?: number;
+};
+
+function validateEditInput(input: EditToolInput): { path: string; edits: Edit[] } {
+	if (!Array.isArray(input.edits) || input.edits.length === 0) {
+		throw new Error("Edit tool input is invalid. edits must contain at least one replacement.");
+	}
+	return { path: input.path, edits: input.edits };
+}
+
+function editAccessError(path: string, error: FileError): Error {
+	return new Error(`Could not edit file: ${path}. Error code: ${error.code}.`, { cause: error });
+}
+
+export function createEditTool(): ToolRegistration<typeof editSchema, EditToolDetails> {
+	return defineTool({
+		name: "edit",
+		description:
+			"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.",
+		parameters: editSchema,
+		prepareArguments: prepareEditArguments,
+		async execute(args, api, context) {
+			const { path, edits } = validateEditInput(args);
+			const env = requireEnv(api);
+			const absolutePath = await resolveToolPath(env, path, context);
+			return withFileMutationQueue(
+				env,
+				absolutePath,
+				async () => {
+					if (context.abortSignal?.aborted) throw new Error("Operation aborted");
+					const info = await env.fileInfo(absolutePath, context);
+					if (!info.ok) throw editAccessError(path, info.error);
+					if (info.value.kind !== "file" && info.value.kind !== "symlink") {
+						throw new Error(`Could not edit file: ${path}. Path is not a file.`);
+					}
+
+					const readResult = await env.readTextFile(absolutePath, context);
+					if (!readResult.ok) throw editAccessError(path, readResult.error);
+					if (context.abortSignal?.aborted) throw new Error("Operation aborted");
+
+					const { bom, text: content } = stripBom(readResult.value);
+					const originalEnding = detectLineEnding(content);
+					const normalizedContent = normalizeToLF(content);
+					const { baseContent, newContent } = applyEditsToNormalizedContent(normalizedContent, edits, path);
+					if (context.abortSignal?.aborted) throw new Error("Operation aborted");
+
+					const finalContent = bom + restoreLineEndings(newContent, originalEnding);
+					const writeResult = await env.writeFile(absolutePath, finalContent, context);
+					if (!writeResult.ok) throw editAccessError(path, writeResult.error);
+					if (context.abortSignal?.aborted) throw new Error("Operation aborted");
+
+					const diffResult = generateDiffString(baseContent, newContent);
+					const details: EditToolDetails = {
+						diff: diffResult.diff,
+						patch: generateUnifiedPatch(path, baseContent, newContent),
+						...(diffResult.firstChangedLine === undefined
+							? {}
+							: { firstChangedLine: diffResult.firstChangedLine }),
+					};
+					return {
+						output: [{ type: "text", text: `Successfully replaced ${edits.length} block(s) in ${path}.` }],
+						details,
+					};
+				},
+				context,
+			);
+		},
+	});
+}

third_party/pi/packages/durable/src/tools/env.ts

@@ -0,0 +1,8 @@
+import type { ExecutionEnv } from "../env/index.ts";
+import type { ToolExecutionApi } from "../harness/types.ts";
+
+/** The call's execution environment; a tool without one fails with an ordinary error result. */
+export function requireEnv(api: ToolExecutionApi): ExecutionEnv {
+	if (api.env === undefined) throw new Error("No execution environment is configured");
+	return api.env;
+}

third_party/pi/packages/durable/src/tools/file-mutation-queue.ts

@@ -0,0 +1,55 @@
+import type { Context } from "@earendil-works/chord";
+import { type ExecutionEnv, getOrThrow } from "../env/index.ts";
+
+/** Tail of the mutation chain of each file, keyed by file system id and canonical path. */
+const queues = new Map<string, Promise<void>>();
+
+async function mutationKey(env: ExecutionEnv, path: string, context: Context): Promise<string> {
+	const absolutePath = getOrThrow(await env.absolutePath(path, context));
+	return `${env.id}\0${await canonical(env, absolutePath, context)}`;
+}
+
+/**
+ * The canonical path; for a file that does not exist yet, its canonical parent joined with its name, so a `write` that
+ * creates a file and a later mutation of it share one key even under a symlinked directory.
+ */
+async function canonical(env: ExecutionEnv, absolutePath: string, context: Context): Promise<string> {
+	const result = await env.canonicalPath(absolutePath, context);
+	if (result.ok) return result.value;
+	if (result.error.code === "not_supported") return absolutePath;
+	if (result.error.code !== "not_found") throw result.error;
+	// The file system splits the path, so a name may contain characters that are separators elsewhere.
+	const parent = getOrThrow(await env.joinPath([absolutePath, ".."], context));
+	if (parent === absolutePath || !absolutePath.startsWith(parent)) return absolutePath;
+	const name = absolutePath.slice(parent.length + (/[/\\]$/.test(parent) ? 0 : 1));
+	return getOrThrow(await env.joinPath([await canonical(env, parent, context), name], context));
+}
+
+/**
+ * Serialize `edit` and `write` mutations of one file within this process: same file system id and canonical path,
+ * whichever environment object the call got. Other files, and other file systems, never wait. Concurrent calls on one
+ * file run in the order their keys resolve. Not a lock against `bash` or other processes.
+ */
+export async function withFileMutationQueue<T>(
+	env: ExecutionEnv,
+	path: string,
+	fn: () => Promise<T>,
+	context: Context,
+): Promise<T> {
+	const key = await mutationKey(env, path, context);
+	// Take the slot without awaiting, so no other call can take it in between.
+	const previous = queues.get(key) ?? Promise.resolve();
+	let release = (): void => {};
+	const done = new Promise<void>((resolve) => {
+		release = resolve;
+	});
+	const tail = previous.then(() => done);
+	queues.set(key, tail);
+	await previous;
+	try {
+		return await fn();
+	} finally {
+		release();
+		if (queues.get(key) === tail) queues.delete(key);
+	}
+}

third_party/pi/packages/durable/src/tools/image-processor.ts

@@ -0,0 +1,58 @@
+/** Limits an image the model sees must keep; the defaults below are the coding agent's. */
+export type ImageLimits = {
+	/** Pixels. Default 2000. */
+	readonly maxWidth: number;
+	/** Pixels. Default 2000. */
+	readonly maxHeight: number;
+	/** Bytes of base64. Default 4.5 MiB, below Anthropic's 5 MB. */
+	readonly maxBytes: number;
+};
+
+export const DEFAULT_IMAGE_LIMITS: ImageLimits = { maxWidth: 2000, maxHeight: 2000, maxBytes: 4.5 * 1024 * 1024 };
+
+/** An image the model can take: a supported format, within the limits, as base64. */
+export type PreparedImage = {
+	readonly data: string;
+	readonly mimeType: string;
+	/** Set when the image was resized: its size before and after, for the model to map coordinates back. */
+	readonly resized?: {
+		readonly from: { readonly width: number; readonly height: number };
+		readonly to: { readonly width: number; readonly height: number };
+	};
+	/** The original format, set when it differs from `mimeType`: BMP becomes PNG, a large PNG may become JPEG. */
+	readonly convertedFrom?: string;
+};
+
+/**
+ * Decodes, orients, resizes, and re-encodes images so the model can take them. `read` uses one when given; without
+ * one, it passes supported images within the byte limit through as they are. `@earendil-works/pi-durable/images`
+ * provides a Photon-based one.
+ */
+export type ImageProcessor = {
+	/**
+	 * The image, prepared to fit `limits`, or `undefined` when it cannot be decoded or made to fit. A promise, so a
+	 * processor can do the work off the event loop.
+	 */
+	prepare(bytes: Uint8Array, mimeType: string, limits: ImageLimits): Promise<PreparedImage | undefined>;
+};
+
+/** Formats every provider takes inline; others, such as BMP, need converting. */
+export const INLINE_IMAGE_TYPES: ReadonlySet<string> = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]);
+
+/**
+ * Standard base64 of `bytes`. `btoa` takes a binary string, built 8 KiB at a time: a single `String.fromCharCode`
+ * call with megabytes of arguments overflows the stack.
+ */
+export function toBase64(bytes: Uint8Array): string {
+	const parts: string[] = [];
+	for (let i = 0; i < bytes.length; i += 8192) {
+		// `apply` takes any array-like, though its type asks for an array.
+		parts.push(String.fromCharCode.apply(null, bytes.subarray(i, i + 8192) as unknown as number[]));
+	}
+	return btoa(parts.join(""));
+}
+
+/** The length of the base64 of `byteLength` bytes. */
+export function base64Length(byteLength: number): number {
+	return Math.ceil(byteLength / 3) * 4;
+}

third_party/pi/packages/durable/src/tools/image.ts

@@ -0,0 +1,182 @@
+const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
+/** Bytes every check except the APNG chunk walk needs: BMP reads up to offset 29. */
+const HEADER_BYTES = 32;
+const BLOCK_BYTES = 64 * 1024;
+
+/** Positional reads of a file of `size` bytes. */
+export type ByteSource = {
+	readonly size: number;
+	read(offset: number, length: number): Promise<Uint8Array>;
+};
+
+/**
+ * `detectSupportedImageMimeType` of a whole file, reading only its header and, for PNG, the chunk headers up to the
+ * first `acTL` or `IDAT`.
+ */
+export async function detectSupportedImageMimeTypeOf(source: ByteSource): Promise<string | undefined> {
+	const header = await source.read(0, HEADER_BYTES);
+	if (!startsWith(header, PNG_SIGNATURE)) return detectSupportedImageMimeType(header);
+	return isPng(header) && !(await isAnimatedPngOf(source)) ? "image/png" : undefined;
+}
+
+/** `isAnimatedPng` over a file read in blocks. */
+async function isAnimatedPngOf(source: ByteSource): Promise<boolean> {
+	let block: Uint8Array = new Uint8Array(0);
+	let blockStart = 0;
+	const bytesAt = async (offset: number, length: number): Promise<Uint8Array> => {
+		if (offset < blockStart || offset + length > blockStart + block.length) {
+			blockStart = offset;
+			block = await source.read(offset, BLOCK_BYTES);
+		}
+		return block.subarray(offset - blockStart, offset - blockStart + length);
+	};
+	let offset = PNG_SIGNATURE.length;
+	while (offset + 8 <= source.size) {
+		const chunkHeader = await bytesAt(offset, 8);
+		const chunkLength = readUint32BE(chunkHeader, 0);
+		if (startsWithAscii(chunkHeader, 4, "acTL")) return true;
+		if (startsWithAscii(chunkHeader, 4, "IDAT")) return false;
+		const nextOffset = offset + 8 + chunkLength + 4;
+		if (nextOffset <= offset || nextOffset > source.size) return false;
+		offset = nextOffset;
+	}
+	return false;
+}
+
+export function detectSupportedImageMimeType(buffer: Uint8Array): string | undefined {
+	if (startsWith(buffer, [0xff, 0xd8, 0xff])) return buffer[3] === 0xf7 ? undefined : "image/jpeg";
+	if (startsWith(buffer, PNG_SIGNATURE)) return isPng(buffer) && !isAnimatedPng(buffer) ? "image/png" : undefined;
+	if (startsWithAscii(buffer, 0, "GIF87a") || startsWithAscii(buffer, 0, "GIF89a")) return "image/gif";
+	if (startsWithAscii(buffer, 0, "RIFF") && startsWithAscii(buffer, 8, "WEBP")) return "image/webp";
+	if (startsWithAscii(buffer, 0, "BM") && isBmp(buffer)) return "image/bmp";
+	return undefined;
+}
+
+function isPng(buffer: Uint8Array): boolean {
+	return (
+		buffer.length >= 16 && readUint32BE(buffer, PNG_SIGNATURE.length) === 13 && startsWithAscii(buffer, 12, "IHDR")
+	);
+}
+
+function isAnimatedPng(buffer: Uint8Array): boolean {
+	let offset = PNG_SIGNATURE.length;
+	while (offset + 8 <= buffer.length) {
+		const chunkLength = readUint32BE(buffer, offset);
+		const chunkTypeOffset = offset + 4;
+		if (startsWithAscii(buffer, chunkTypeOffset, "acTL")) return true;
+		if (startsWithAscii(buffer, chunkTypeOffset, "IDAT")) return false;
+		const nextOffset = offset + 8 + chunkLength + 4;
+		if (nextOffset <= offset || nextOffset > buffer.length) return false;
+		offset = nextOffset;
+	}
+	return false;
+}
+
+/**
+ * The width and height an image of `mimeType` declares in its header, without decoding it: PNG's IHDR, GIF's screen,
+ * WebP's VP8, VP8L, or VP8X chunk, JPEG's first SOF segment. `undefined` when the header cannot be read.
+ */
+export function imageDimensions(
+	bytes: Uint8Array,
+	mimeType: string,
+): { readonly width: number; readonly height: number } | undefined {
+	switch (mimeType) {
+		case "image/png":
+			return bytes.length < 24 ? undefined : { width: readUint32BE(bytes, 16), height: readUint32BE(bytes, 20) };
+		case "image/gif":
+			return bytes.length < 10 ? undefined : { width: readUint16LE(bytes, 6), height: readUint16LE(bytes, 8) };
+		case "image/webp":
+			if (bytes.length < 30) return undefined;
+			if (startsWithAscii(bytes, 12, "VP8 "))
+				return { width: readUint16LE(bytes, 26) & 0x3fff, height: readUint16LE(bytes, 28) & 0x3fff };
+			if (startsWithAscii(bytes, 12, "VP8L")) {
+				const bits = readUint32LE(bytes, 21);
+				return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 };
+			}
+			if (startsWithAscii(bytes, 12, "VP8X")) {
+				const uint24 = (offset: number): number => readUint16LE(bytes, offset) + ((bytes[offset + 2] ?? 0) << 16);
+				return { width: uint24(24) + 1, height: uint24(27) + 1 };
+			}
+			return undefined;
+		case "image/jpeg":
+			for (let offset = 2; offset + 9 <= bytes.length; ) {
+				if (bytes[offset] !== 0xff) return undefined;
+				const marker = bytes[offset + 1]!;
+				if (marker === 0xff) {
+					offset++;
+					continue;
+				}
+				// SOF0 to SOF15, except DHT (C4), JPG (C8), and DAC (CC), which share the range.
+				if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
+					return { width: readUint16BE(bytes, offset + 7), height: readUint16BE(bytes, offset + 5) };
+				}
+				offset += 2 + readUint16BE(bytes, offset + 2);
+			}
+			return undefined;
+		default:
+			return undefined;
+	}
+}
+
+function isBmp(buffer: Uint8Array): boolean {
+	if (buffer.length < 26) return false;
+	const declaredFileSize = readUint32LE(buffer, 2);
+	const pixelDataOffset = readUint32LE(buffer, 10);
+	const dibHeaderSize = readUint32LE(buffer, 14);
+	if (declaredFileSize !== 0 && declaredFileSize < 26) return false;
+	if (pixelDataOffset < 14 + dibHeaderSize) return false;
+	if (declaredFileSize !== 0 && pixelDataOffset >= declaredFileSize) return false;
+
+	let colorPlanes: number;
+	let bitsPerPixel: number;
+	if (dibHeaderSize === 12) {
+		colorPlanes = readUint16LE(buffer, 22);
+		bitsPerPixel = readUint16LE(buffer, 24);
+	} else if (dibHeaderSize >= 40 && dibHeaderSize <= 124) {
+		if (buffer.length < 30) return false;
+		colorPlanes = readUint16LE(buffer, 26);
+		bitsPerPixel = readUint16LE(buffer, 28);
+	} else {
+		return false;
+	}
+	return colorPlanes === 1 && [1, 4, 8, 16, 24, 32].includes(bitsPerPixel);
+}
+
+function readUint16LE(buffer: Uint8Array, offset: number): number {
+	return (buffer[offset] ?? 0) + ((buffer[offset + 1] ?? 0) << 8);
+}
+
+function readUint16BE(buffer: Uint8Array, offset: number): number {
+	return ((buffer[offset] ?? 0) << 8) + (buffer[offset + 1] ?? 0);
+}
+
+function readUint32BE(buffer: Uint8Array, offset: number): number {
+	return (
+		(buffer[offset] ?? 0) * 0x1000000 +
+		((buffer[offset + 1] ?? 0) << 16) +
+		((buffer[offset + 2] ?? 0) << 8) +
+		(buffer[offset + 3] ?? 0)
+	);
+}
+
+function readUint32LE(buffer: Uint8Array, offset: number): number {
+	return (
+		(buffer[offset] ?? 0) +
+		((buffer[offset + 1] ?? 0) << 8) +
+		((buffer[offset + 2] ?? 0) << 16) +
+		(buffer[offset + 3] ?? 0) * 0x1000000
+	);
+}
+
+function startsWith(buffer: Uint8Array, bytes: number[]): boolean {
+	if (buffer.length < bytes.length) return false;
+	return bytes.every((byte, index) => buffer[index] === byte);
+}
+
+function startsWithAscii(buffer: Uint8Array, offset: number, text: string): boolean {
+	if (buffer.length < offset + text.length) return false;
+	for (let index = 0; index < text.length; index++) {
+		if (buffer[offset + index] !== text.charCodeAt(index)) return false;
+	}
+	return true;
+}

third_party/pi/packages/durable/src/tools/index.ts

@@ -0,0 +1,39 @@
+import { defineExtension } from "../harness/define.ts";
+import { createBashTool } from "./bash.ts";
+import { createEditTool } from "./edit.ts";
+import { createReadTool, type ReadToolOptions } from "./read.ts";
+import { createWriteTool } from "./write.ts";
+
+export {
+	type BashExecution,
+	type BashPrepare,
+	type BashToolInput,
+	type BashToolOptions,
+	createBashTool,
+	createPowerShellTool,
+	type PowerShellToolInput,
+	type PowerShellToolOptions,
+} from "./bash.ts";
+export { createEditTool, type EditToolDetails, type EditToolInput } from "./edit.ts";
+export {
+	DEFAULT_IMAGE_LIMITS,
+	type ImageLimits,
+	type ImageProcessor,
+	type PreparedImage,
+} from "./image-processor.ts";
+export { createReadTool, type ReadToolDetails, type ReadToolInput, type ReadToolOptions } from "./read.ts";
+export { createWriteTool, type WriteToolInput } from "./write.ts";
+
+/**
+ * The `coding-tools` extension: `read`, `write`, `edit`, and `bash`; nothing installs it automatically.
+ * `createPowerShellTool()` adds `powershell`. `images` prepares the images `read` returns (see `ReadToolOptions`).
+ */
+export function createCodingTools(options: ReadToolOptions = {}) {
+	return defineExtension({
+		name: "coding-tools",
+		tools: [createReadTool(options), createWriteTool(), createEditTool(), createBashTool()],
+	});
+}
+
+/** `createCodingTools()` without an image processor. */
+export const CodingTools = createCodingTools();

third_party/pi/packages/durable/src/tools/path-utils.ts

@@ -0,0 +1,30 @@
+import type { Context } from "@earendil-works/chord";
+import { type ExecutionEnv, getOrThrow } from "../env/index.ts";
+
+const UNICODE_SPACES = /[\u00A0\u2000-\u200A\u202F\u205F\u3000]/g;
+const NARROW_NO_BREAK_SPACE = "\u202F";
+
+function normalizeToolPath(path: string): string {
+	const normalized = path.replace(UNICODE_SPACES, " ");
+	return normalized.startsWith("@") ? normalized.slice(1) : normalized;
+}
+
+export async function resolveToolPath(env: ExecutionEnv, path: string, context: Context): Promise<string> {
+	return getOrThrow(await env.absolutePath(normalizeToolPath(path), context));
+}
+
+export async function resolveReadToolPath(env: ExecutionEnv, path: string, context: Context): Promise<string> {
+	const resolved = await resolveToolPath(env, path, context);
+	const variants = [
+		resolved,
+		resolved.replace(/ (AM|PM)\./gi, `${NARROW_NO_BREAK_SPACE}$1.`),
+		resolved.normalize("NFD"),
+		resolved.replace(/'/g, "\u2019"),
+		resolved.normalize("NFD").replace(/'/g, "\u2019"),
+	];
+
+	for (const variant of new Set(variants)) {
+		if (getOrThrow(await env.exists(variant, context))) return variant;
+	}
+	return resolved;
+}

third_party/pi/packages/durable/src/tools/read.ts

@@ -0,0 +1,312 @@
+import type { Context } from "@earendil-works/chord";
+import { characterEnd } from "@earendil-works/chord/output";
+import { type Static, Type } from "typebox";
+import {
+	type BinaryReader,
+	type FileInfo,
+	getOrThrow,
+	type LineScan,
+	rangeDecoder,
+	startsWithBom,
+} from "../env/index.ts";
+import { defineTool } from "../harness/define.ts";
+import type { ToolDiagnostic, ToolExecutionApi, ToolExecutionResult, ToolRegistration } from "../harness/types.ts";
+import {
+	DEFAULT_MAX_BYTES,
+	DEFAULT_MAX_LINES,
+	formatSize,
+	type TruncationResult,
+	truncateHeadOf,
+	utf8ByteLength,
+} from "../truncate.ts";
+import { requireEnv } from "./env.ts";
+import { detectSupportedImageMimeTypeOf, imageDimensions } from "./image.ts";
+import {
+	base64Length,
+	DEFAULT_IMAGE_LIMITS,
+	type ImageLimits,
+	type ImageProcessor,
+	INLINE_IMAGE_TYPES,
+	toBase64,
+} from "./image-processor.ts";
+import { resolveReadToolPath } from "./path-utils.ts";
+
+const readSchema = Type.Object({
+	path: Type.String({ description: "Path to the file to read (relative or absolute)" }),
+	offset: Type.Optional(Type.Number({ description: "Line number to start reading from (1-indexed)" })),
+	limit: Type.Optional(Type.Number({ description: "Maximum number of lines to read" })),
+});
+
+export type ReadToolInput = Static<typeof readSchema>;
+
+export type ReadToolDetails = {
+	/** How the shown text was cut; the text itself is the result content. */
+	truncation?: Omit<TruncationResult, "content">;
+};
+
+const READ_CHUNK = 64 * 1024;
+
+/** `Array.prototype.slice`'s conversion of an index: NaN is 0, other values truncate toward zero. */
+function sliceIndex(value: number): number {
+	return Number.isNaN(value) ? 0 : Math.trunc(value);
+}
+
+/**
+ * The decoded start of bytes `[start, end)` of the file, decoded as part of the whole file: all of it, or enough for
+ * `truncateHeadOf` (more than `DEFAULT_MAX_BYTES + 1` bytes, or `DEFAULT_MAX_LINES` newlines).
+ */
+async function readHead(
+	reader: BinaryReader,
+	start: number,
+	end: number,
+	skipBom: boolean,
+	context: Context,
+): Promise<string> {
+	const decoder = rangeDecoder();
+	let text = "";
+	let newlines = 0;
+	for (let position = skipBom && start === 0 ? 3 : start; position < end; ) {
+		const bytes = getOrThrow(await reader.read(position, Math.min(READ_CHUNK, end - position), context));
+		if (bytes.length === 0) break;
+		position += bytes.length;
+		const decoded = decoder.decode(bytes, { stream: true });
+		text += decoded;
+		for (let index = decoded.indexOf("\n"); index !== -1; index = decoded.indexOf("\n", index + 1)) newlines++;
+		if (newlines >= DEFAULT_MAX_LINES || utf8ByteLength(text) > DEFAULT_MAX_BYTES + 1) return text;
+	}
+	return text + decoder.decode();
+}
+
+export type ReadToolOptions = {
+	/** Resizes and converts images; without one, images within `DEFAULT_IMAGE_LIMITS.maxBytes` pass through as they are. */
+	readonly images?: ImageProcessor;
+};
+
+/**
+ * Reads text files and images. Remarks about truncation, continuation, and image processing are diagnostics; the content
+ * is only file text, or one image, which programs get as its `ImageContent`.
+ */
+export function createReadTool(options: ReadToolOptions = {}): ToolRegistration<typeof readSchema, ReadToolDetails> {
+	const formats = options.images === undefined ? "jpg, png, gif, webp" : "jpg, png, gif, webp, bmp";
+	return defineTool({
+		name: "read",
+		description: `Read the contents of a file. Supports text files and images (${formats}). Images are sent as attachments. For text files, output is truncated to ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`,
+		parameters: readSchema,
+		async execute(args, api, context) {
+			const { path, offset, limit } = args;
+			const env = requireEnv(api);
+			const absolutePath = await resolveReadToolPath(env, path, context);
+			const reader = getOrThrow(await env.openBinaryReader(absolutePath, undefined, context));
+			try {
+				// A concurrent writer can change the file between the scan and the reads. Appending (a growing log) leaves
+				// the scanned text as it was; an image must be whole. Otherwise the file is read again once.
+				for (let attempt = 0; ; attempt++) {
+					const before = getOrThrow(await reader.info(context));
+					const mimeType = await detectSupportedImageMimeTypeOf({
+						size: before.size,
+						read: async (position, length) => getOrThrow(await reader.read(position, length, context)),
+					});
+					const result =
+						mimeType === undefined
+							? await readText(reader, path, offset, limit, context)
+							: await readImage(
+									reader,
+									before,
+									path,
+									mimeType,
+									options.images,
+									await imageModel(api, context),
+									context,
+								);
+					const after = getOrThrow(await reader.info(context));
+					const unchanged = after.size === before.size && after.mtimeMs === before.mtimeMs;
+					if (unchanged || (mimeType === undefined && after.size > before.size)) return result;
+					if (attempt === 1) throw new Error(`${path} changed while it was read`);
+				}
+			} finally {
+				await reader.close(context);
+			}
+		},
+	});
+}
+
+/**
+ * One image block, prepared by `processor` or, without one, as it is when its format is inline and it fits the limits
+ * (its size is checked before reading it, its dimensions from its header); otherwise an error result saying why. What happened to the image is an `info` diagnostic, so programs get the
+ * bare `ImageContent`. A model without image input sees pi-ai's placeholder instead; a diagnostic says so.
+ */
+async function readImage(
+	reader: BinaryReader,
+	info: FileInfo,
+	path: string,
+	mimeType: string,
+	processor: ImageProcessor | undefined,
+	{ vision, limits }: ImageModel,
+	context: Context,
+): Promise<ToolExecutionResult<ReadToolDetails>> {
+	let image: { readonly data: string; readonly mimeType: string };
+	const notes: string[] = [];
+	if (processor !== undefined) {
+		const bytes = getOrThrow(await reader.read(0, info.size, context));
+		const prepared = await processor.prepare(bytes, mimeType, limits);
+		if (prepared === undefined)
+			return imageError(`${path} is an image (${mimeType}) that cannot be prepared for the model`);
+		image = prepared;
+		if (prepared.convertedFrom !== undefined)
+			notes.push(`Converted from ${prepared.convertedFrom} to ${prepared.mimeType}.`);
+		if (prepared.resized !== undefined) {
+			const { from, to } = prepared.resized;
+			const scale = (from.width / to.width).toFixed(2);
+			notes.push(
+				`Resized from ${from.width}x${from.height} to ${to.width}x${to.height}. Multiply coordinates by ${scale} to map them to the original.`,
+			);
+		}
+	} else {
+		if (!INLINE_IMAGE_TYPES.has(mimeType)) {
+			return imageError(
+				`${path} is an image (${mimeType}) that needs converting, and no image processor is configured`,
+			);
+		}
+		const unshrinkable = "and no image processor is configured to shrink it";
+		if (base64Length(info.size) > limits.maxBytes) {
+			return imageError(
+				`${path} is an image (${mimeType}) of ${formatSize(info.size)}, too large to send, ${unshrinkable}`,
+			);
+		}
+		const bytes = getOrThrow(await reader.read(0, info.size, context));
+		const size = imageDimensions(bytes, mimeType);
+		if (size === undefined) return imageError(`${path} is an image (${mimeType}) whose size cannot be read`);
+		if (size.width > limits.maxWidth || size.height > limits.maxHeight) {
+			return imageError(
+				`${path} is an image (${mimeType}) of ${size.width}x${size.height}, larger than ${limits.maxWidth}x${limits.maxHeight}, ${unshrinkable}`,
+			);
+		}
+		image = { data: toBase64(bytes), mimeType };
+	}
+	if (!vision) notes.push("The current model does not support images; it sees a placeholder instead.");
+	const diagnostics: ToolDiagnostic[] = [
+		{ severity: "info", code: "image", message: [`Read image file [${image.mimeType}].`, ...notes].join(" ") },
+	];
+	return { output: [{ type: "image", data: image.data, mimeType: image.mimeType }], diagnostics };
+}
+
+/** What the conversation's model takes: whether images, and within which limits. */
+type ImageModel = { readonly vision: boolean; readonly limits: ImageLimits };
+
+/**
+ * The conversation model's image input: `DEFAULT_IMAGE_LIMITS` overridden by its `inputLimits.images.resize`. Without
+ * a resolvable model, it counts as one that takes images within the defaults.
+ */
+async function imageModel(api: ToolExecutionApi, context: Context): Promise<ImageModel> {
+	const ref = (await api.agent(context)).model;
+	const model = ref === undefined ? undefined : api.models.getModel(ref.provider, ref.modelId);
+	const resize = model?.inputLimits?.images?.resize;
+	return {
+		vision: model?.input.includes("image") ?? true,
+		limits: {
+			maxWidth: resize?.maxWidth ?? DEFAULT_IMAGE_LIMITS.maxWidth,
+			maxHeight: resize?.maxHeight ?? DEFAULT_IMAGE_LIMITS.maxHeight,
+			maxBytes: resize?.maxBytes ?? DEFAULT_IMAGE_LIMITS.maxBytes,
+		},
+	};
+}
+
+function imageError(message: string): ToolExecutionResult<ReadToolDetails> {
+	return { output: [], isError: true, diagnostics: [{ severity: "error", code: "unsupported_image", message }] };
+}
+
+/**
+ * The read result for the opened file. It equals decoding the whole file with `TextDecoder`, splitting it on `\n`,
+ * and bounding the selected lines with `truncateHead`, while reading only one scan's worth of the file plus the head.
+ */
+async function readText(
+	reader: BinaryReader,
+	path: string,
+	offset: number | undefined,
+	limit: number | undefined,
+	context: Context,
+) {
+	const startLine = offset ? Math.max(0, offset - 1) : 0;
+	const startLineDisplay = startLine + 1;
+	// Lines are selected like `allLines.slice(startLine, endLine)`, which truncates fractional indices.
+	const sliceStart = sliceIndex(startLine);
+	// One pass finds the line count and the selection; a selection past the last line ends with it, as `slice` does,
+	// and an empty one (a zero or negative limit) is scanned as one line and then ignored.
+	// A start beyond any file is scanned from 0 only to count lines; the offset check below then fails as before.
+	const scanStart = Number.isSafeInteger(sliceStart) ? sliceStart : 0;
+	const requestedEnd = limit === undefined ? undefined : Math.max(scanStart + 1, sliceIndex(startLine + limit));
+	const scanEnd = requestedEnd !== undefined && Number.isSafeInteger(requestedEnd) ? requestedEnd : undefined;
+	const scanOf = async (endLine: number | undefined): Promise<LineScan> =>
+		getOrThrow(
+			await reader.scanLines({ startLine: scanStart, ...(endLine === undefined ? {} : { endLine }) }, context),
+		);
+	let scan = await scanOf(scanEnd);
+	const totalFileLines = scan.newlines + 1;
+	if (startLine >= totalFileLines) {
+		throw new Error(`Offset ${offset} is beyond end of file (${totalFileLines} lines total)`);
+	}
+
+	let userLimitedLines: number | undefined;
+	let selectedLineCount = totalFileLines - sliceStart;
+	if (limit !== undefined) {
+		const endLine = Math.min(startLine + limit, totalFileLines);
+		userLimitedLines = endLine - startLine;
+		// `slice` counts a negative end from the end of the lines, which only the line count tells; scan again for it.
+		const relativeEnd = sliceIndex(endLine);
+		const sliceEnd = relativeEnd < 0 ? Math.max(totalFileLines + relativeEnd, 0) : relativeEnd;
+		selectedLineCount = Math.max(0, sliceEnd - sliceStart);
+		if (selectedLineCount > 0 && relativeEnd < 0) scan = await scanOf(sliceEnd);
+	}
+	const empty = selectedLineCount === 0;
+	// Counted like `truncateHead`: a trailing newline adds no line, and empty text has none.
+	const endsWithNewline = !empty && scan.lastLineStart === scan.end && scan.lastLineStart > scan.start;
+	const totals = {
+		lines: empty || scan.selectedBytes === 0 ? 0 : selectedLineCount - (endsWithNewline ? 1 : 0),
+		bytes: empty ? 0 : scan.selectedBytes,
+	};
+	const firstBytes = getOrThrow(await reader.read(0, 3, context));
+	const head = empty ? "" : await readHead(reader, scan.start, scan.end, startsWithBom(firstBytes), context);
+
+	const { content: headText, ...truncation } = truncateHeadOf(head, totals);
+	const diagnostics: ToolDiagnostic[] = [];
+	let outputText = headText;
+	let details: ReadToolDetails | undefined;
+	if (truncation.firstLineExceedsLimit) {
+		// Show the start of the line, cut at the byte limit on a character boundary. Like `allLines[startLine]`, a
+		// fractional start line names no line.
+		const integral = Number.isInteger(startLine);
+		const lineBytes = new TextEncoder().encode(integral ? (head.split("\n")[0] ?? "") : "");
+		const lineSize = integral ? scan.firstLineBytes : 0;
+		const end = characterEnd(lineBytes, DEFAULT_MAX_BYTES);
+		outputText = new TextDecoder().decode(lineBytes.subarray(0, end));
+		diagnostics.push({
+			severity: "warn",
+			code: "truncated",
+			message: `Line ${startLineDisplay} is ${formatSize(lineSize)}, exceeds the ${formatSize(DEFAULT_MAX_BYTES)} limit; showing its first ${formatSize(end)}. Use bash: sed -n '${startLineDisplay}p' ${path} | tail -c +${end + 1}`,
+		});
+		details = { truncation: { ...truncation, outputBytes: end, outputLines: 1 } };
+	} else if (truncation.truncated) {
+		const endLineDisplay = startLineDisplay + truncation.outputLines - 1;
+		const nextOffset = endLineDisplay + 1;
+		const limitText = truncation.truncatedBy === "lines" ? "" : ` (${formatSize(DEFAULT_MAX_BYTES)} limit)`;
+		diagnostics.push({
+			severity: "info",
+			code: "truncated",
+			message: `Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}${limitText}. Use offset=${nextOffset} to continue.`,
+		});
+		details = { truncation };
+	} else if (userLimitedLines !== undefined && startLine + userLimitedLines < totalFileLines) {
+		const remaining = totalFileLines - (startLine + userLimitedLines);
+		const nextOffset = startLine + userLimitedLines + 1;
+		diagnostics.push({
+			severity: "info",
+			message: `${remaining} more lines in file. Use offset=${nextOffset} to continue.`,
+		});
+	}
+
+	return {
+		output: outputText === "" ? [] : [{ type: "text" as const, text: outputText }],
+		...(details === undefined ? {} : { details }),
+		diagnostics,
+	};
+}

third_party/pi/packages/durable/src/tools/write.ts

@@ -0,0 +1,39 @@
+import { type Static, Type } from "typebox";
+import { getOrThrow } from "../env/index.ts";
+import { defineTool } from "../harness/define.ts";
+import type { ToolRegistration } from "../harness/types.ts";
+import { requireEnv } from "./env.ts";
+import { withFileMutationQueue } from "./file-mutation-queue.ts";
+import { resolveToolPath } from "./path-utils.ts";
+
+const writeSchema = Type.Object({
+	path: Type.String({ description: "Path to the file to write (relative or absolute)" }),
+	content: Type.String({ description: "Content to write to the file" }),
+});
+
+export type WriteToolInput = Static<typeof writeSchema>;
+
+export function createWriteTool(): ToolRegistration<typeof writeSchema> {
+	return defineTool({
+		name: "write",
+		description:
+			"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.",
+		parameters: writeSchema,
+		async execute(args, api, context) {
+			const { path, content } = args;
+			const env = requireEnv(api);
+			const absolutePath = await resolveToolPath(env, path, context);
+			return withFileMutationQueue(
+				env,
+				absolutePath,
+				async () => {
+					if (context.abortSignal?.aborted) throw new Error("Operation aborted");
+					getOrThrow(await env.writeFile(absolutePath, content, context));
+					if (context.abortSignal?.aborted) throw new Error("Operation aborted");
+					return { output: [{ type: "text", text: `Successfully wrote to ${path}` }] };
+				},
+				context,
+			);
+		},
+	});
+}

third_party/pi/packages/durable/src/truncate.ts

@@ -0,0 +1,174 @@
+import { utf8ByteLength } from "@earendil-works/chord/output";
+/**
+ * Shared truncation utilities for tool outputs.
+ *
+ * Truncation is based on two independent limits - whichever is hit first wins:
+ * - Line limit (default: 2000 lines)
+ * - Byte limit (default: 50KB)
+ *
+ * Never returns partial lines. Tool output streams are bounded by `OutputBuffer` from `@earendil-works/chord/output` instead.
+ */
+
+export const DEFAULT_MAX_LINES = 2000;
+export const DEFAULT_MAX_BYTES = 50 * 1024; // 50KB
+
+export interface TruncationResult {
+	/** The truncated content */
+	content: string;
+	/** Whether truncation occurred */
+	truncated: boolean;
+	/** Which limit was hit: "lines", "bytes", or null if not truncated */
+	truncatedBy: "lines" | "bytes" | null;
+	/** Total number of lines in the original content */
+	totalLines: number;
+	/** Total number of bytes in the original content */
+	totalBytes: number;
+	/** Number of complete lines in the truncated output */
+	outputLines: number;
+	/** Number of bytes in the truncated output */
+	outputBytes: number;
+	/** Whether the last line was partially truncated (only for tail truncation edge case) */
+	lastLinePartial: boolean;
+	/** Whether the first line exceeded the byte limit (for head truncation) */
+	firstLineExceedsLimit: boolean;
+	/** The max lines limit that was applied */
+	maxLines: number;
+	/** The max bytes limit that was applied */
+	maxBytes: number;
+}
+
+export interface TruncationOptions {
+	/** Maximum number of lines (default: 2000) */
+	maxLines?: number;
+	/** Maximum number of bytes (default: 50KB) */
+	maxBytes?: number;
+}
+
+// One implementation, shared with the output buffer.
+export { utf8ByteLength };
+
+function splitLinesForCounting(content: string): string[] {
+	if (content.length === 0) return [];
+	const lines = content.split("\n");
+	if (content.endsWith("\n")) lines.pop();
+	return lines;
+}
+
+/**
+ * Format bytes as human-readable size.
+ */
+export function formatSize(bytes: number): string {
+	if (bytes < 1024) {
+		return `${bytes}B`;
+	} else if (bytes < 1024 * 1024) {
+		return `${(bytes / 1024).toFixed(1)}KB`;
+	} else {
+		return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+	}
+}
+
+/**
+ * Truncate content from the head (keep first N lines/bytes).
+ * Suitable for file reads where you want to see the beginning.
+ *
+ * Never returns partial lines. If first line exceeds byte limit,
+ * returns empty content with firstLineExceedsLimit=true.
+ */
+export function truncateHead(content: string, options: TruncationOptions = {}): TruncationResult {
+	return truncateHeadOf(
+		content,
+		{ lines: splitLinesForCounting(content).length, bytes: utf8ByteLength(content) },
+		options,
+	);
+}
+
+/**
+ * `truncateHead` of a text known by a prefix and its totals (`lines` counted like `truncateHead`, ignoring a trailing
+ * newline). The prefix must be the whole text, or longer than `maxBytes + 1` UTF-8 bytes, or hold at least `maxLines`
+ * newlines; then the result equals `truncateHead` of the whole text.
+ */
+export function truncateHeadOf(
+	prefix: string,
+	totals: { readonly lines: number; readonly bytes: number },
+	options: TruncationOptions = {},
+): TruncationResult {
+	const maxLines = options.maxLines ?? DEFAULT_MAX_LINES;
+	const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
+
+	const totalBytes = totals.bytes;
+	const lines = splitLinesForCounting(prefix);
+	const totalLines = totals.lines;
+
+	// Check if no truncation needed
+	if (totalLines <= maxLines && totalBytes <= maxBytes) {
+		return {
+			content: prefix,
+			truncated: false,
+			truncatedBy: null,
+			totalLines,
+			totalBytes,
+			outputLines: totalLines,
+			outputBytes: totalBytes,
+			lastLinePartial: false,
+			firstLineExceedsLimit: false,
+			maxLines,
+			maxBytes,
+		};
+	}
+
+	// Check if first line alone exceeds byte limit
+	const firstLineBytes = utf8ByteLength(lines[0]);
+	if (firstLineBytes > maxBytes) {
+		return {
+			content: "",
+			truncated: true,
+			truncatedBy: "bytes",
+			totalLines,
+			totalBytes,
+			outputLines: 0,
+			outputBytes: 0,
+			lastLinePartial: false,
+			firstLineExceedsLimit: true,
+			maxLines,
+			maxBytes,
+		};
+	}
+
+	// Collect complete lines that fit
+	const outputLinesArr: string[] = [];
+	let outputBytesCount = 0;
+	let truncatedBy: "lines" | "bytes" = "lines";
+
+	for (let i = 0; i < lines.length && i < maxLines; i++) {
+		const line = lines[i];
+		const lineBytes = utf8ByteLength(line) + (i > 0 ? 1 : 0); // +1 for newline
+
+		if (outputBytesCount + lineBytes > maxBytes) {
+			truncatedBy = "bytes";
+			break;
+		}
+
+		outputLinesArr.push(line);
+		outputBytesCount += lineBytes;
+	}
+
+	// Without a byte break, only omitted lines prove the line limit was reached; otherwise a trailing newline exceeded bytes.
+	if (truncatedBy !== "bytes") truncatedBy = outputLinesArr.length < totalLines ? "lines" : "bytes";
+
+	const outputContent = outputLinesArr.join("\n");
+	const finalOutputBytes = utf8ByteLength(outputContent);
+
+	return {
+		content: outputContent,
+		truncated: true,
+		truncatedBy,
+		totalLines,
+		totalBytes,
+		outputLines: outputLinesArr.length,
+		outputBytes: finalOutputBytes,
+		lastLinePartial: false,
+		firstLineExceedsLimit: false,
+		maxLines,
+		maxBytes,
+	};
+}