chore(vendor): import pi 1.3.0

Imported as a linear vendor commit so forge rebases do not replay the upstream root tree.

git-subtree-dir: third_party/pi

git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0

third_party/pi/packages/durable/src/storage/jsonl/index.ts

@@ -0,0 +1,6 @@
+export {
+	JsonlCorruptionError,
+	JsonlStorage,
+	type JsonlStorageOptions,
+	JsonlStoragePoisonedError,
+} from "./storage.ts";

third_party/pi/packages/durable/src/storage/jsonl/node.ts

@@ -0,0 +1,14 @@
+import type { Context } from "@earendil-works/chord";
+import { NodeExecutionEnv } from "../../env/node.ts";
+import { JsonlStorage, type JsonlStorageOptions } from "./storage.ts";
+
+/** Open or create a JSONL storage directory using the local Node filesystem. */
+export async function openNodeJsonlStorage(
+	directory: string,
+	context: Context,
+	options: JsonlStorageOptions = {},
+): Promise<JsonlStorage> {
+	return JsonlStorage.open(directory, new NodeExecutionEnv({ cwd: process.cwd() }), context, options);
+}
+
+export { JsonlStorage, type JsonlStorageOptions } from "./storage.ts";

third_party/pi/packages/durable/src/storage/jsonl/storage.ts

@@ -0,0 +1,846 @@
+import type { Context, JsonValue } from "@earendil-works/chord";
+import type { FileError, FileSystem } from "../../env/index.ts";
+import { StorageRequestError } from "../../errors.ts";
+import { idFromNumber, seqFromNumber } from "../../ids.ts";
+import type {
+	ConversationId,
+	ConversationQuery,
+	Cursor,
+	DocumentAddress,
+	DocumentContent,
+	DocumentCreate,
+	DocumentId,
+	DocumentPoint,
+	DocumentQuery,
+	EntryId,
+	EntryQuery,
+	Id,
+	Seq,
+	Storage,
+	StorageWrite,
+	SubmissionId,
+	SubmissionQuery,
+	TaskId,
+	TaskQuery,
+	TaskRecord,
+} from "../../types.ts";
+import { MemoryStorage } from "../memory.ts";
+
+const FORMAT_VERSION = 1;
+const MAIN_FILE = "main.jsonl";
+const RECLAIM_SUFFIX = ".reclaim";
+const textDecoder = new TextDecoder("utf-8", { fatal: true });
+
+type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
+
+type MainOperation =
+	| Extract<StorageWrite, { readonly type: "conversation" | "entry" | "submission" | "document.retire" }>
+	| { readonly type: "task"; readonly value: StoredTask }
+	| { readonly type: "task.sidecar"; readonly id: TaskId; readonly ordinal: number }
+	| { readonly type: "document.create"; readonly record: DocumentCreate; readonly ordinal: number }
+	| { readonly type: "document.change"; readonly id: DocumentId; readonly ordinal: number };
+
+type MainMarker = {
+	readonly format: typeof FORMAT_VERSION;
+	readonly type: "commit";
+	readonly seq: Seq;
+	readonly writes: readonly MainOperation[];
+};
+
+type SidecarPayload =
+	| { readonly type: "task"; readonly value: StoredTask }
+	| { readonly type: "document"; readonly id: DocumentId; readonly content: DocumentContent };
+
+type SidecarRecord = {
+	readonly format: typeof FORMAT_VERSION;
+	readonly type: "record";
+	readonly seq: Seq;
+	readonly ordinal: number;
+	readonly payload: SidecarPayload;
+};
+
+type ParsedLine<T> = {
+	readonly value: T;
+	readonly start: number;
+};
+
+type ParsedFile<T> = {
+	readonly path: string;
+	readonly lines: readonly ParsedLine<T>[];
+};
+
+type EncodedCommit = {
+	readonly marker: string;
+	readonly sidecars: ReadonlyMap<string, string>;
+};
+
+export type JsonlStorageOptions = {
+	/** Flush every affected sidecar before appending the main marker. Defaults to false. */
+	readonly fsync?: boolean;
+};
+
+export class JsonlCorruptionError extends Error {
+	constructor(message: string, cause?: Error) {
+		super(message, cause === undefined ? undefined : { cause });
+		this.name = "JsonlCorruptionError";
+	}
+}
+
+export class JsonlStoragePoisonedError extends Error {
+	constructor(cause: Error) {
+		super("JSONL storage is poisoned and must be reopened", { cause });
+		this.name = "JsonlStoragePoisonedError";
+	}
+}
+
+const isObject = (value: unknown): value is Record<string, unknown> =>
+	value !== null && typeof value === "object" && !Array.isArray(value);
+
+const isSafeInteger = (value: unknown): value is number => Number.isSafeInteger(value);
+
+const sidecarFileName = (kind: "doc" | "task", id: DocumentId | TaskId): string => `${kind}-${id}.jsonl`;
+
+const isSidecarFileName = (name: string): boolean => /^(?:doc|task)-(?:0|[1-9]\d*)\.jsonl$/.test(name);
+
+const isReclaimFileName = (name: string): boolean => /^(?:doc|task)-(?:0|[1-9]\d*)\.jsonl\.reclaim$/.test(name);
+
+const isCurrentOnly = (record: DocumentCreate): boolean =>
+	record.scope.kind !== "conversation" || record.history === "latest";
+
+const sidecarKey = (file: string, seq: Seq, ordinal: number): string => JSON.stringify([file, seq, ordinal]);
+
+const jsonLine = (value: MainMarker | SidecarRecord): string => `${JSON.stringify(value)}\n`;
+
+const errorFromFile = (action: string, error: FileError): Error =>
+	new Error(`JSONL ${action} failed: ${error.message}`, { cause: error });
+
+const parseJson = (text: string, description: string): unknown => {
+	try {
+		return JSON.parse(text) as unknown;
+	} catch (error) {
+		throw new JsonlCorruptionError(`Malformed complete ${description}`, error instanceof Error ? error : undefined);
+	}
+};
+
+const validateMainOperation = (value: unknown, description: string): MainOperation => {
+	if (!isObject(value) || typeof value.type !== "string") {
+		throw new JsonlCorruptionError(`Invalid write in ${description}`);
+	}
+	switch (value.type) {
+		case "conversation":
+		case "entry":
+		case "submission":
+			if (!isObject(value.value) || !isSafeInteger(value.value.id)) {
+				throw new JsonlCorruptionError(`Invalid ${value.type} write in ${description}`);
+			}
+			return value as MainOperation;
+		case "task":
+			if (
+				!isObject(value.value) ||
+				!isSafeInteger(value.value.id) ||
+				!isObject(value.value.state) ||
+				value.value.state.status !== "terminal"
+			) {
+				throw new JsonlCorruptionError(`Invalid terminal task write in ${description}`);
+			}
+			return value as MainOperation;
+		case "document.retire":
+			if (!isSafeInteger(value.id)) throw new JsonlCorruptionError(`Invalid document retirement in ${description}`);
+			return value as MainOperation;
+		case "task.sidecar":
+			if (!isSafeInteger(value.id) || !isSafeInteger(value.ordinal) || value.ordinal < 0) {
+				throw new JsonlCorruptionError(`Invalid task sidecar write in ${description}`);
+			}
+			return value as MainOperation;
+		case "document.create":
+			if (
+				!isObject(value.record) ||
+				!isSafeInteger(value.record.id) ||
+				!isSafeInteger(value.ordinal) ||
+				value.ordinal < 0
+			) {
+				throw new JsonlCorruptionError(`Invalid document creation in ${description}`);
+			}
+			return value as MainOperation;
+		case "document.change":
+			if (!isSafeInteger(value.id) || !isSafeInteger(value.ordinal) || value.ordinal < 0) {
+				throw new JsonlCorruptionError(`Invalid document change in ${description}`);
+			}
+			return value as MainOperation;
+		default:
+			throw new JsonlCorruptionError(`Unknown write type in ${description}`);
+	}
+};
+
+const parseMainMarker = (text: string, line: number): MainMarker => {
+	const description = `${MAIN_FILE} line ${line}`;
+	const value = parseJson(text, description);
+	if (
+		!isObject(value) ||
+		value.format !== FORMAT_VERSION ||
+		value.type !== "commit" ||
+		!isSafeInteger(value.seq) ||
+		value.seq < 1 ||
+		!Array.isArray(value.writes)
+	) {
+		throw new JsonlCorruptionError(`Invalid commit marker in ${description}`);
+	}
+	return {
+		format: FORMAT_VERSION,
+		type: "commit",
+		seq: seqFromNumber(value.seq),
+		writes: value.writes.map((write) => validateMainOperation(write, description)),
+	};
+};
+
+const validateDocumentContent = (value: unknown, description: string): DocumentContent => {
+	if (!isObject(value) || !isSafeInteger(value.version) || value.version < 1) {
+		throw new JsonlCorruptionError(`Invalid document content in ${description}`);
+	}
+	if (value.kind === "base" && isObject(value.value)) return value as DocumentContent;
+	if (value.kind === "delta" && Array.isArray(value.ops)) return value as DocumentContent;
+	throw new JsonlCorruptionError(`Invalid document content in ${description}`);
+};
+
+const parseSidecarRecord = (text: string, file: string, line: number): SidecarRecord => {
+	const description = `${file} line ${line}`;
+	const value = parseJson(text, description);
+	if (
+		!isObject(value) ||
+		value.format !== FORMAT_VERSION ||
+		value.type !== "record" ||
+		!isSafeInteger(value.seq) ||
+		value.seq < 1 ||
+		!isSafeInteger(value.ordinal) ||
+		value.ordinal < 0 ||
+		!isObject(value.payload) ||
+		typeof value.payload.type !== "string"
+	) {
+		throw new JsonlCorruptionError(`Invalid sidecar record in ${description}`);
+	}
+	if (value.payload.type === "task") {
+		if (
+			!isObject(value.payload.value) ||
+			!isSafeInteger(value.payload.value.id) ||
+			!isObject(value.payload.value.state) ||
+			value.payload.value.state.status === "terminal"
+		) {
+			throw new JsonlCorruptionError(`Invalid live task record in ${description}`);
+		}
+	} else if (value.payload.type === "document") {
+		if (!isSafeInteger(value.payload.id)) {
+			throw new JsonlCorruptionError(`Invalid document record in ${description}`);
+		}
+		validateDocumentContent(value.payload.content, description);
+	} else {
+		throw new JsonlCorruptionError(`Unknown sidecar record type in ${description}`);
+	}
+	return value as SidecarRecord;
+};
+
+/** Portable JSONL implementation of the storage contract. */
+export class JsonlStorage implements Storage {
+	private readonly fs: FileSystem;
+	private readonly directory: string;
+	private readonly mainPath: string;
+	private readonly fsync: boolean;
+	private readonly memory = new MemoryStorage();
+	private readonly currentOnlyDocuments = new Set<DocumentId>();
+	private readonly liveTaskSidecars = new Set<TaskId>();
+	private closed = false;
+	private poisonError: JsonlStoragePoisonedError | undefined;
+
+	private constructor(fs: FileSystem, directory: string, mainPath: string, options: JsonlStorageOptions) {
+		this.fs = fs;
+		this.directory = directory;
+		this.mainPath = mainPath;
+		this.fsync = options.fsync ?? false;
+	}
+
+	/** Open or create a JSONL storage directory using the supplied filesystem. */
+	static async open(
+		directory: string,
+		fs: FileSystem,
+		context: Context,
+		options: JsonlStorageOptions = {},
+	): Promise<JsonlStorage> {
+		const absolute = await fs.absolutePath(directory, context);
+		if (!absolute.ok) throw errorFromFile("path resolution", absolute.error);
+		const created = await fs.createDir(absolute.value, { recursive: true }, context);
+		if (!created.ok) throw errorFromFile("directory creation", created.error);
+		const mainPathResult = await fs.joinPath([absolute.value, MAIN_FILE], context);
+		if (!mainPathResult.ok) throw errorFromFile("path join", mainPathResult.error);
+		const storage = new JsonlStorage(fs, absolute.value, mainPathResult.value, options);
+		await storage.recover(context);
+		return storage;
+	}
+
+	async commit(writes: readonly StorageWrite[], context: Context): Promise<Seq> {
+		this.assertUsable();
+		const prepared = this.memory.prepareCommit(writes);
+		const encoded = this.encodeCommit(prepared.seq, prepared.writes);
+		const reclamations = this.planReclamations(prepared.writes, encoded);
+		const sidecars = await Promise.all(
+			[...encoded.sidecars].map(async ([file, content]) => ({
+				file,
+				content,
+				path: await this.resolveFile(file, context),
+			})),
+		);
+
+		for (const sidecar of sidecars) {
+			const result = await this.fs.appendFile(sidecar.path, sidecar.content, context);
+			if (!result.ok) throw this.poison(errorFromFile(`append to ${sidecar.file}`, result.error));
+		}
+		if (this.fsync) {
+			for (const sidecar of sidecars) {
+				const result = await this.fs.flushFile(sidecar.path, context);
+				if (!result.ok) throw this.poison(errorFromFile(`flush of ${sidecar.file}`, result.error));
+			}
+		}
+		const marker = await this.fs.appendFile(this.mainPath, encoded.marker, context);
+		if (!marker.ok) throw this.poison(errorFromFile(`append to ${MAIN_FILE}`, marker.error));
+		const seq = prepared.apply();
+		this.adoptSidecarState(prepared.writes);
+		await this.reclaimSidecars(reclamations, context);
+		return seq;
+	}
+
+	async mintId<I extends Id<string>>(): Promise<I> {
+		return this.store.mintId<I>();
+	}
+
+	async conversation(id: ConversationId, context: Context) {
+		return this.store.conversation(id, context);
+	}
+
+	async scanConversations(query: ConversationQuery, limit: number, cursor: Cursor | undefined, context: Context) {
+		return this.store.scanConversations(query, limit, cursor, context);
+	}
+
+	entry(id: EntryId, context: Context): ReturnType<Storage["entry"]>;
+	entry(conversationId: ConversationId, id: EntryId, context: Context): ReturnType<Storage["entry"]>;
+	async entry(idOrConversationId: EntryId | ConversationId, idOrContext: EntryId | Context, context?: Context) {
+		if (context === undefined)
+			return this.store.entry(idFromNumber<EntryId>(idOrConversationId), idOrContext as Context);
+		if (typeof idOrContext !== "number") throw new StorageRequestError("Storage.entry() requires an entry ID");
+		return this.store.entry(
+			idFromNumber<ConversationId>(idOrConversationId),
+			idFromNumber<EntryId>(idOrContext),
+			context,
+		);
+	}
+
+	async findLatestHeadMarker(
+		conversationId: ConversationId,
+		atOrBeforeEntryId: EntryId | undefined,
+		context: Context,
+	) {
+		return this.store.findLatestHeadMarker(conversationId, atOrBeforeEntryId, context);
+	}
+
+	async scanEntries(query: EntryQuery, limit: number, cursor: Cursor | undefined, context: Context) {
+		return this.store.scanEntries(query, limit, cursor, context);
+	}
+
+	async task(id: TaskId, context: Context) {
+		return this.store.task(id, context);
+	}
+
+	async scanTasks(query: TaskQuery, limit: number, cursor: Cursor | undefined, context: Context) {
+		return this.store.scanTasks(query, limit, cursor, context);
+	}
+
+	async submission(id: SubmissionId, context: Context) {
+		return this.store.submission(id, context);
+	}
+
+	async scanSubmissions(query: SubmissionQuery, limit: number, cursor: Cursor | undefined, context: Context) {
+		return this.store.scanSubmissions(query, limit, cursor, context);
+	}
+
+	async submissionByRequest(conversationId: ConversationId, requestId: string, context: Context) {
+		return this.store.submissionByRequest(conversationId, requestId, context);
+	}
+
+	async findDocument(address: DocumentAddress, at: DocumentPoint, context: Context) {
+		return this.store.findDocument(address, at, context);
+	}
+
+	async document(id: DocumentId, at: DocumentPoint, context: Context) {
+		return this.store.document(id, at, context);
+	}
+
+	async scanDocuments(query: DocumentQuery, limit: number, cursor: Cursor | undefined, context: Context) {
+		return this.store.scanDocuments(query, limit, cursor, context);
+	}
+
+	async close(context: Context): Promise<void> {
+		if (this.closed) return;
+		this.closed = true;
+		await this.memory.close(context);
+	}
+
+	private encodeCommit(seq: Seq, writes: readonly StorageWrite[]): EncodedCommit {
+		const mainWrites: MainOperation[] = [];
+		const records = new Map<string, SidecarRecord[]>();
+		let nextOrdinal = 0;
+		const addSidecar = (file: string, payload: SidecarPayload): number => {
+			const ordinal = nextOrdinal++;
+			let fileRecords = records.get(file);
+			if (fileRecords === undefined) {
+				fileRecords = [];
+				records.set(file, fileRecords);
+			}
+			fileRecords.push({ format: FORMAT_VERSION, type: "record", seq, ordinal, payload });
+			return ordinal;
+		};
+
+		for (const write of writes) {
+			switch (write.type) {
+				case "conversation":
+				case "entry":
+				case "submission":
+				case "document.retire":
+					mainWrites.push(write);
+					break;
+				case "task":
+					if (write.value.state.status === "terminal") mainWrites.push(write);
+					else {
+						const ordinal = addSidecar(sidecarFileName("task", write.value.id), {
+							type: "task",
+							value: write.value,
+						});
+						mainWrites.push({ type: "task.sidecar", id: write.value.id, ordinal });
+					}
+					break;
+				case "document.create": {
+					const ordinal = addSidecar(sidecarFileName("doc", write.record.id), {
+						type: "document",
+						id: write.record.id,
+						content: write.content,
+					});
+					mainWrites.push({ type: "document.create", record: write.record, ordinal });
+					break;
+				}
+				case "document.change": {
+					const ordinal = addSidecar(sidecarFileName("doc", write.id), {
+						type: "document",
+						id: write.id,
+						content: write.content,
+					});
+					mainWrites.push({ type: "document.change", id: write.id, ordinal });
+					break;
+				}
+			}
+		}
+
+		const sidecars = new Map<string, string>();
+		for (const [file, fileRecords] of records) {
+			sidecars.set(file, fileRecords.map((record) => jsonLine(record)).join(""));
+		}
+		const marker: MainMarker = {
+			format: FORMAT_VERSION,
+			type: "commit",
+			seq,
+			writes: mainWrites,
+		};
+		return { marker: jsonLine(marker), sidecars };
+	}
+
+	private planReclamations(writes: readonly StorageWrite[], encoded: EncodedCommit): ReadonlyMap<string, string> {
+		const createdCurrentOnlyDocuments = new Set<DocumentId>();
+		const retiredDocuments = new Set<DocumentId>();
+		const baseDocuments = new Set<DocumentId>();
+		const finalTasks = new Map<TaskId, StoredTask>();
+		for (const write of writes) {
+			switch (write.type) {
+				case "document.create":
+					if (isCurrentOnly(write.record)) createdCurrentOnlyDocuments.add(write.record.id);
+					break;
+				case "document.change":
+					if (write.content.kind === "base") baseDocuments.add(write.id);
+					break;
+				case "document.retire":
+					retiredDocuments.add(write.id);
+					break;
+				case "task":
+					finalTasks.set(write.value.id, write.value);
+					break;
+			}
+		}
+
+		const replacements = new Map<string, string>();
+		const isCurrentOnlyDocument = (id: DocumentId): boolean =>
+			this.currentOnlyDocuments.has(id) || createdCurrentOnlyDocuments.has(id);
+		for (const id of retiredDocuments) {
+			if (isCurrentOnlyDocument(id)) replacements.set(sidecarFileName("doc", id), "");
+		}
+		for (const id of baseDocuments) {
+			if (!isCurrentOnlyDocument(id) || retiredDocuments.has(id)) continue;
+			const file = sidecarFileName("doc", id);
+			const content = encoded.sidecars.get(file);
+			if (content !== undefined) replacements.set(file, content);
+		}
+		for (const [id, task] of finalTasks) {
+			if (
+				task.state.status === "terminal" &&
+				(this.liveTaskSidecars.has(id) || encoded.sidecars.has(sidecarFileName("task", id)))
+			) {
+				replacements.set(sidecarFileName("task", id), "");
+			}
+		}
+		return replacements;
+	}
+
+	private adoptSidecarState(writes: readonly StorageWrite[]): void {
+		for (const write of writes) {
+			if (write.type === "document.create") {
+				if (isCurrentOnly(write.record)) this.currentOnlyDocuments.add(write.record.id);
+			} else if (write.type === "task") {
+				if (write.value.state.status === "terminal") this.liveTaskSidecars.delete(write.value.id);
+				else this.liveTaskSidecars.add(write.value.id);
+			}
+		}
+	}
+
+	/** The marker already published this state, so reclamation is retryable best-effort maintenance. */
+	private async reclaimSidecars(replacements: ReadonlyMap<string, string>, context: Context): Promise<void> {
+		if (replacements.size === 0) return;
+		if (this.fsync) {
+			const flushed = await this.fs.flushFile(this.mainPath, context);
+			if (!flushed.ok) return;
+		}
+		for (const [file, content] of replacements) await this.replaceSidecar(file, content, context);
+	}
+
+	private async replaceSidecar(file: string, content: string, context: Context): Promise<void> {
+		const path = await this.fs.joinPath([this.directory, file], context);
+		if (!path.ok) return;
+		if (content === "") {
+			await this.fs.remove(path.value, { force: true }, context);
+			return;
+		}
+		const temporaryPath = await this.fs.joinPath([this.directory, `${file}${RECLAIM_SUFFIX}`], context);
+		if (!temporaryPath.ok) return;
+		const written = await this.fs.writeFile(temporaryPath.value, content, context);
+		if (!written.ok) return;
+		if (this.fsync) {
+			const flushed = await this.fs.flushFile(temporaryPath.value, context);
+			if (!flushed.ok) return;
+		}
+		await this.fs.renameFile(temporaryPath.value, path.value, context);
+	}
+
+	private async recover(context: Context): Promise<void> {
+		const fs = this.fs;
+		const directory = this.directory;
+		const memory = this.memory;
+		const main = await JsonlStorage.readLines(fs, this.mainPath, MAIN_FILE, context, (text, line) =>
+			parseMainMarker(text, line),
+		);
+		let previousSeq = 0;
+		for (const marker of main.lines) {
+			if (marker.value.seq <= previousSeq) {
+				throw new JsonlCorruptionError(`Commit sequence does not strictly increase in ${MAIN_FILE}`);
+			}
+			previousSeq = marker.value.seq;
+		}
+
+		const listed = await fs.listDir(directory, context);
+		if (!listed.ok) throw errorFromFile("directory listing", listed.error);
+		for (const info of listed.value) {
+			if (info.kind === "file" && isReclaimFileName(info.name)) {
+				await fs.remove(info.path, { force: true }, context);
+			}
+		}
+		const sidecarFiles = listed.value
+			.filter((info) => info.kind === "file" && isSidecarFileName(info.name))
+			.map((info) => info.name)
+			.sort();
+		const parsedFiles = new Map<string, ParsedFile<SidecarRecord>>();
+		const recordByKey = new Map<string, ParsedLine<SidecarRecord>>();
+		for (const file of sidecarFiles) {
+			const pathResult = await fs.joinPath([directory, file], context);
+			if (!pathResult.ok) throw errorFromFile("path join", pathResult.error);
+			const parsed = await JsonlStorage.readLines(fs, pathResult.value, file, context, (text, line) =>
+				parseSidecarRecord(text, file, line),
+			);
+			parsedFiles.set(file, parsed);
+			let previous: SidecarRecord | undefined;
+			for (const line of parsed.lines) {
+				if (
+					previous !== undefined &&
+					(line.value.seq < previous.seq ||
+						(line.value.seq === previous.seq && line.value.ordinal <= previous.ordinal))
+				) {
+					throw new JsonlCorruptionError(`Sidecar records are out of order in ${file}`);
+				}
+				previous = line.value;
+				recordByKey.set(sidecarKey(file, line.value.seq, line.value.ordinal), line);
+			}
+		}
+
+		const currentOnlyDocuments = new Set<DocumentId>();
+		const retiredDocuments = new Set<DocumentId>();
+		const finalTaskIsLive = new Map<TaskId, boolean>();
+		for (const { value: marker } of main.lines) {
+			for (const operation of marker.writes) {
+				if (operation.type === "document.create") {
+					if (isCurrentOnly(operation.record)) currentOnlyDocuments.add(operation.record.id);
+				} else if (operation.type === "document.retire") {
+					retiredDocuments.add(operation.id);
+				} else if (operation.type === "task") {
+					finalTaskIsLive.set(operation.value.id, false);
+				} else if (operation.type === "task.sidecar") {
+					finalTaskIsLive.set(operation.id, true);
+				}
+			}
+		}
+		const retiredCurrentOnlyDocuments = new Set([...retiredDocuments].filter((id) => currentOnlyDocuments.has(id)));
+
+		const latestBases = new Map<DocumentId, SidecarRecord>();
+		for (const { value: marker } of main.lines) {
+			for (const operation of marker.writes) {
+				if (operation.type !== "document.create" && operation.type !== "document.change") continue;
+				const id = operation.type === "document.create" ? operation.record.id : operation.id;
+				if (!currentOnlyDocuments.has(id)) continue;
+				const record = recordByKey.get(
+					sidecarKey(sidecarFileName("doc", id), marker.seq, operation.ordinal),
+				)?.value;
+				if (
+					record?.payload.type !== "document" ||
+					record.payload.id !== id ||
+					record.payload.content.kind !== "base"
+				) {
+					continue;
+				}
+				const previous = latestBases.get(id);
+				if (
+					previous === undefined ||
+					record.seq > previous.seq ||
+					(record.seq === previous.seq && record.ordinal > previous.ordinal)
+				) {
+					latestBases.set(id, record);
+				}
+			}
+		}
+
+		const isBeforeLatestBase = (id: DocumentId, seq: Seq, ordinal: number): boolean => {
+			const base = latestBases.get(id);
+			return base !== undefined && (seq < base.seq || (seq === base.seq && ordinal < base.ordinal));
+		};
+		const terminalTasks = new Set([...finalTaskIsLive].filter(([, live]) => !live).map(([id]) => id));
+		const confirmed = new Set<string>();
+		for (const line of main.lines) {
+			const marker = line.value;
+			const writes: StorageWrite[] = [];
+			for (const operation of marker.writes) {
+				switch (operation.type) {
+					case "conversation":
+					case "entry":
+					case "submission":
+					case "task":
+					case "document.retire":
+						writes.push(operation);
+						break;
+					case "task.sidecar": {
+						const optional = terminalTasks.has(operation.id);
+						const record = JsonlStorage.confirmRecord(
+							marker,
+							operation.ordinal,
+							sidecarFileName("task", operation.id),
+							recordByKey,
+							confirmed,
+							optional,
+						);
+						if (record !== undefined) {
+							if (record.payload.type !== "task" || record.payload.value.id !== operation.id) {
+								throw new JsonlCorruptionError(
+									`Confirmed task sidecar data does not match commit ${marker.seq}`,
+								);
+							}
+							if (!optional) writes.push({ type: "task", value: record.payload.value });
+						}
+						break;
+					}
+					case "document.create":
+					case "document.change": {
+						const id = operation.type === "document.create" ? operation.record.id : operation.id;
+						const reclaimed =
+							retiredCurrentOnlyDocuments.has(id) || isBeforeLatestBase(id, marker.seq, operation.ordinal);
+						const record = JsonlStorage.confirmRecord(
+							marker,
+							operation.ordinal,
+							sidecarFileName("doc", id),
+							recordByKey,
+							confirmed,
+							reclaimed,
+						);
+						let content: DocumentContent | undefined;
+						if (record !== undefined) {
+							if (record.payload.type !== "document" || record.payload.id !== id) {
+								throw new JsonlCorruptionError(
+									`Confirmed document sidecar data does not match commit ${marker.seq}`,
+								);
+							}
+							content = record.payload.content;
+						}
+						if (operation.type === "document.create") {
+							if (content !== undefined && content.kind !== "base") {
+								throw new JsonlCorruptionError(
+									`Document creation lacks a confirmed base in commit ${marker.seq}`,
+								);
+							}
+							writes.push({
+								type: "document.create",
+								record: operation.record,
+								content: reclaimed || content === undefined ? { kind: "base", version: 1, value: {} } : content,
+							});
+						} else if (!reclaimed && content !== undefined) {
+							writes.push({ type: "document.change", id, content });
+						}
+						break;
+					}
+				}
+			}
+			try {
+				memory.prepareCommit(writes, marker.seq).apply();
+			} catch (error) {
+				throw new JsonlCorruptionError(
+					`Invalid committed state at sequence ${marker.seq}`,
+					error instanceof Error ? error : undefined,
+				);
+			}
+		}
+
+		const reclamations = new Map<string, string>();
+		for (const [file, parsed] of parsedFiles) {
+			let unconfirmedAt: number | undefined;
+			for (const line of parsed.lines) {
+				const key = sidecarKey(file, line.value.seq, line.value.ordinal);
+				if (confirmed.has(key)) {
+					if (unconfirmedAt !== undefined) {
+						throw new JsonlCorruptionError(`Confirmed record follows an unconfirmed tail in ${file}`);
+					}
+				} else if (unconfirmedAt === undefined) {
+					unconfirmedAt = line.start;
+				}
+			}
+			if (unconfirmedAt !== undefined) {
+				const truncated = await fs.truncateFile(parsed.path, unconfirmedAt, context);
+				if (!truncated.ok) throw errorFromFile(`tail truncation of ${file}`, truncated.error);
+			}
+
+			const numericId = Number(file.slice(file.indexOf("-") + 1, -".jsonl".length));
+			const confirmedLines = parsed.lines.filter((line) =>
+				confirmed.has(sidecarKey(file, line.value.seq, line.value.ordinal)),
+			);
+			let retainedLines: readonly ParsedLine<SidecarRecord>[] | undefined;
+			if (file.startsWith("task-") && terminalTasks.has(idFromNumber<TaskId>(numericId))) {
+				retainedLines = [];
+			} else if (file.startsWith("doc-")) {
+				const documentId = idFromNumber<DocumentId>(numericId);
+				if (retiredCurrentOnlyDocuments.has(documentId)) {
+					retainedLines = [];
+				} else if (latestBases.has(documentId)) {
+					retainedLines = confirmedLines.filter(
+						(line) => !isBeforeLatestBase(documentId, line.value.seq, line.value.ordinal),
+					);
+				}
+			}
+			if (
+				retainedLines !== undefined &&
+				(retainedLines.length < confirmedLines.length || retainedLines.length === 0)
+			) {
+				reclamations.set(file, retainedLines.map((line) => jsonLine(line.value)).join(""));
+			}
+		}
+		await this.reclaimSidecars(reclamations, context);
+
+		for (const id of currentOnlyDocuments) this.currentOnlyDocuments.add(id);
+		for (const [id, live] of finalTaskIsLive) {
+			if (live) this.liveTaskSidecars.add(id);
+		}
+	}
+
+	private static confirmRecord(
+		marker: MainMarker,
+		ordinal: number,
+		file: string,
+		recordByKey: ReadonlyMap<string, ParsedLine<SidecarRecord>>,
+		confirmed: Set<string>,
+		optional: boolean,
+	): SidecarRecord | undefined {
+		const key = sidecarKey(file, marker.seq, ordinal);
+		if (confirmed.has(key)) throw new JsonlCorruptionError(`Sidecar record is confirmed more than once`);
+		const line = recordByKey.get(key);
+		if (line === undefined) {
+			if (optional) return undefined;
+			throw new JsonlCorruptionError(`Missing confirmed sidecar record ${file} at sequence ${marker.seq}`);
+		}
+		confirmed.add(key);
+		return line.value;
+	}
+
+	private static async readLines<T>(
+		fs: FileSystem,
+		path: string,
+		name: string,
+		context: Context,
+		parse: (text: string, line: number) => T,
+	): Promise<ParsedFile<T>> {
+		const read = await fs.readBinaryFile(path, context);
+		if (!read.ok) {
+			if (read.error.code === "not_found") return { path, lines: [] };
+			throw errorFromFile(`read of ${name}`, read.error);
+		}
+		const bytes = read.value;
+		let completeSize = bytes.length;
+		if (completeSize > 0 && bytes[completeSize - 1] !== 0x0a) {
+			completeSize = bytes.lastIndexOf(0x0a) + 1;
+			const truncated = await fs.truncateFile(path, completeSize, context);
+			if (!truncated.ok) throw errorFromFile(`torn-line truncation of ${name}`, truncated.error);
+		}
+		const lines: ParsedLine<T>[] = [];
+		let start = 0;
+		let lineNumber = 1;
+		for (let end = 0; end < completeSize; end++) {
+			if (bytes[end] !== 0x0a) continue;
+			let text: string;
+			try {
+				text = textDecoder.decode(bytes.subarray(start, end));
+			} catch (error) {
+				throw new JsonlCorruptionError(
+					`Invalid UTF-8 in complete ${name} line ${lineNumber}`,
+					error instanceof Error ? error : undefined,
+				);
+			}
+			lines.push({ value: parse(text, lineNumber), start });
+			start = end + 1;
+			lineNumber++;
+		}
+		return { path, lines };
+	}
+
+	private async resolveFile(file: string, context: Context): Promise<string> {
+		const path = await this.fs.joinPath([this.directory, file], context);
+		if (!path.ok) throw errorFromFile("path join", path.error);
+		return path.value;
+	}
+
+	private get store(): MemoryStorage {
+		this.assertUsable();
+		return this.memory;
+	}
+
+	private poison(cause: Error): JsonlStoragePoisonedError {
+		this.poisonError ??= new JsonlStoragePoisonedError(cause);
+		return this.poisonError;
+	}
+
+	private assertUsable(): void {
+		if (this.closed) throw new Error("JsonlStorage is closed");
+		if (this.poisonError !== undefined) throw this.poisonError;
+	}
+}

third_party/pi/packages/durable/src/storage/memory.ts

@@ -0,0 +1,864 @@
+import type { Context, JsonValue } from "@earendil-works/chord";
+import { applyImmutableBatches, type Op } from "@earendil-works/chord/delta";
+import { StorageRequestError } from "../errors.ts";
+import { idFromNumber, seqFromNumber } from "../ids.ts";
+import type {
+	ConversationId,
+	ConversationQuery,
+	ConversationRecord,
+	Cursor,
+	DocumentAddress,
+	DocumentContent,
+	DocumentCreate,
+	DocumentId,
+	DocumentPoint,
+	DocumentQuery,
+	DocumentRecord,
+	EntryId,
+	EntryQuery,
+	EntryRecord,
+	Id,
+	JsonObject,
+	Page,
+	ScanOrder,
+	Seq,
+	Storage,
+	StorageWrite,
+	StoredDocument,
+	SubmissionId,
+	SubmissionQuery,
+	SubmissionRecord,
+	TaskId,
+	TaskQuery,
+	TaskRecord,
+} from "../types.ts";
+import { nextCursor, type ScanStart, scanStart } from "./scan.ts";
+
+type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
+type TaskStatus = StoredTask["state"]["status"];
+type SubmissionStatus = SubmissionRecord["status"];
+type TableName = "conversation" | "entry" | "task" | "submission" | "document";
+type DocumentRevision = DocumentContent & { readonly seq: Seq };
+type StoredDocumentState = {
+	record: DocumentRecord;
+	revisions: DocumentRevision[];
+};
+type DocumentAction = {
+	create?: DocumentCreate;
+	content?: DocumentContent;
+	retire: boolean;
+};
+
+function* documentDeltaBatches(
+	id: DocumentId,
+	version: number,
+	revisions: readonly DocumentRevision[],
+	start: number,
+): Generator<readonly Op[]> {
+	for (let index = start; index < revisions.length; index++) {
+		const revision = revisions[index]!;
+		if (revision.kind !== "delta" || revision.version !== version) {
+			throw new Error(`Document ${id} crosses a stored version boundary without a base`);
+		}
+		yield revision.ops;
+	}
+}
+
+type DocumentAddressIndex = {
+	ids: DocumentId[];
+	currentId?: DocumentId;
+};
+
+type State = {
+	recordTypes: Map<Id<string>, TableName>;
+	conversations: Map<ConversationId, ConversationRecord>;
+	conversationIds: ConversationId[];
+	conversationIdsByOwnerConversation: Map<ConversationId, ConversationId[]>;
+	conversationIdsByOwnerTask: Map<TaskId, ConversationId[]>;
+	entries: Map<EntryId, EntryRecord>;
+	entryIds: Map<ConversationId, EntryId[]>;
+	headEntryIds: Map<ConversationId, EntryId[]>;
+	entryCommitSeqs: Map<EntryId, Seq>;
+	tasks: Map<TaskId, StoredTask>;
+	taskIds: TaskId[];
+	taskIdsByStatus: Record<TaskStatus, TaskId[]>;
+	submissions: Map<SubmissionId, SubmissionRecord>;
+	submissionIds: SubmissionId[];
+	submissionIdsByStatus: Record<SubmissionStatus, SubmissionId[]>;
+	submissionIdsByRequest: Map<ConversationId, Map<string, SubmissionId>>;
+	documents: Map<DocumentId, StoredDocumentState>;
+	documentAddresses: Map<string, DocumentAddressIndex>;
+	documentIdsByScope: Map<string, DocumentId[]>;
+};
+
+const clone = <T>(value: T): T => {
+	if (value === null || typeof value !== "object") return value;
+	if (Array.isArray(value)) return value.map((item) => clone(item)) as T;
+	const source = value as Record<string, unknown>;
+	const nullPrototype = Object.getPrototypeOf(value) === null;
+	const result = (nullPrototype ? Object.create(null) : {}) as Record<string, unknown>;
+	for (const key of Object.keys(source)) {
+		const copied = clone(source[key]);
+		if (!nullPrototype && key in result) {
+			Object.defineProperty(result, key, {
+				value: copied,
+				writable: true,
+				enumerable: true,
+				configurable: true,
+			});
+		} else {
+			result[key] = copied;
+		}
+	}
+	return result as T;
+};
+
+const freeze = <T>(value: T): T => {
+	if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value;
+	for (const child of Object.values(value)) freeze(child);
+	return Object.freeze(value);
+};
+
+const cursorId = <I extends Id<string>>(cursor: Readonly<Record<string, JsonValue>> | undefined): I | undefined => {
+	const after = cursor?.after;
+	if (after === undefined) return undefined;
+	if (typeof after !== "number" || !Number.isSafeInteger(after))
+		throw new StorageRequestError("Invalid storage cursor");
+	return idFromNumber<I>(after);
+};
+
+const lowerBound = <I extends Id<string>>(ids: readonly I[], target: number): number => {
+	let low = 0;
+	let high = ids.length;
+	while (low < high) {
+		const middle = (low + high) >>> 1;
+		if (ids[middle] < target) low = middle + 1;
+		else high = middle;
+	}
+	return low;
+};
+
+const upperBound = <I extends Id<string>>(ids: readonly I[], target: number): number => {
+	let low = 0;
+	let high = ids.length;
+	while (low < high) {
+		const middle = (low + high) >>> 1;
+		if (ids[middle] <= target) low = middle + 1;
+		else high = middle;
+	}
+	return low;
+};
+
+const insertSorted = <I extends Id<string>>(ids: I[], id: I): void => {
+	if (ids.length === 0 || ids[ids.length - 1] < id) ids.push(id);
+	else ids.splice(lowerBound(ids, id), 0, id);
+};
+
+const removeSorted = <I extends Id<string>>(ids: I[], id: I): void => {
+	const index = lowerBound(ids, id);
+	if (ids[index] === id) ids.splice(index, 1);
+};
+
+const insertMapId = <K, I extends Id<string>>(index: Map<K, I[]>, key: K, id: I): void => {
+	let ids = index.get(key);
+	if (ids === undefined) {
+		ids = [];
+		index.set(key, ids);
+	}
+	insertSorted(ids, id);
+};
+
+const scopeKey = (scope: DocumentRecord["scope"]): string => {
+	switch (scope.kind) {
+		case "session":
+			return JSON.stringify(["session"]);
+		case "conversation":
+			return JSON.stringify(["conversation", scope.conversationId]);
+		case "task":
+			return JSON.stringify(["task", scope.taskId]);
+	}
+};
+
+const addressKey = (address: DocumentAddress): string =>
+	JSON.stringify([
+		address.kind,
+		scopeKey(address.scope),
+		address.key === undefined ? ["singleton"] : ["family", address.key],
+	]);
+
+const recordAddressKey = (record: DocumentRecord | DocumentCreate): string =>
+	addressKey({ kind: record.kind, scope: record.scope, key: record.key });
+
+const isAliveAt = (record: DocumentRecord, at: DocumentPoint): boolean => {
+	if (at === "current") return record.retiredAt === undefined;
+	return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt);
+};
+
+const isCurrentOnly = (record: DocumentRecord): boolean =>
+	record.scope.kind !== "conversation" || record.history === "latest";
+
+const tableContaining = (state: State, id: Id<string>): TableName | undefined => state.recordTypes.get(id);
+
+const page = <T extends { readonly id: Id<string> }>(
+	values: readonly T[],
+	limit: number,
+	order: ScanOrder = "ascending",
+): Page<T, Cursor> => {
+	const items = values.slice(0, limit);
+	if (values.length <= limit) return { items: clone(items) };
+	return { items: clone(items), next: nextCursor(items.at(-1)!.id, order) };
+};
+
+/** Indexes of sorted `ids` in scan order, after the cursor's ID when there is one. */
+function* scanIndexes(ids: readonly Id<string>[], start: ScanStart): Generator<number> {
+	if (start.order === "ascending") {
+		const first = start.after === undefined ? 0 : upperBound(ids, start.after);
+		for (let index = first; index < ids.length; index++) yield index;
+	} else {
+		const end = start.after === undefined ? ids.length : lowerBound(ids, start.after);
+		for (let index = end - 1; index >= 0; index--) yield index;
+	}
+}
+
+/** A fully validated, detached state mutation whose application performs no fallible preparation. */
+export interface PreparedMemoryCommit {
+	readonly seq: Seq;
+	/** Deeply frozen detached writes for persistence. */
+	readonly writes: readonly StorageWrite[];
+	apply(): Seq;
+}
+
+/**
+ * Detached in-memory reference implementation of `Storage`.
+ *
+ * Reads and retained writes are cloned intentionally to match the ownership boundary
+ * of serialization-backed stores. This is backend conformance, not validation.
+ */
+export class MemoryStorage implements Storage {
+	private readonly state: State = {
+		recordTypes: new Map(),
+		conversations: new Map(),
+		conversationIds: [],
+		conversationIdsByOwnerConversation: new Map(),
+		conversationIdsByOwnerTask: new Map(),
+		entries: new Map(),
+		entryIds: new Map(),
+		headEntryIds: new Map(),
+		entryCommitSeqs: new Map(),
+		tasks: new Map(),
+		taskIds: [],
+		taskIdsByStatus: { pending: [], running: [], waiting: [], completing: [], terminal: [] },
+		submissions: new Map(),
+		submissionIds: [],
+		submissionIdsByStatus: { queued: [], placed: [], done: [], unanswered: [] },
+		submissionIdsByRequest: new Map(),
+		documents: new Map(),
+		documentAddresses: new Map(),
+		documentIdsByScope: new Map(),
+	};
+	private nextId = 2;
+	private nextSeq = 1;
+	private closed = false;
+
+	async commit(writes: readonly StorageWrite[], _context: Context): Promise<Seq> {
+		return this.prepareCommit(writes).apply();
+	}
+
+	/** Validate and detach one commit without changing observable state. */
+	prepareCommit(writes: readonly StorageWrite[], seq: Seq = seqFromNumber(this.nextSeq)): PreparedMemoryCommit {
+		this.assertOpen();
+		if (!Number.isSafeInteger(seq) || seq < this.nextSeq) {
+			throw new Error(`Commit sequence ${seq} does not strictly increase`);
+		}
+		const detachedWrites = freeze(this.resolveDocumentCopies(writes.map((write) => clone(write))));
+		this.checkGlobalIds(detachedWrites);
+		const documentActions = this.prepareDocumentActions(detachedWrites);
+		this.checkDocumentActions(documentActions);
+		let applied = false;
+		return {
+			seq,
+			writes: detachedWrites,
+			apply: () => {
+				if (!applied) {
+					applied = true;
+					this.applyPreparedCommit(detachedWrites, documentActions, seq);
+				}
+				return seq;
+			},
+		};
+	}
+
+	private resolveDocumentCopies(writes: StorageWrite[]): StorageWrite[] {
+		if (!writes.some((write) => write.type === "document.copy")) return writes;
+		const changedDocumentIds = new Set<DocumentId>();
+		for (const write of writes) {
+			if (write.type === "document.create" || write.type === "document.copy") {
+				changedDocumentIds.add(write.record.id);
+			} else if (write.type === "document.change" || write.type === "document.retire") {
+				changedDocumentIds.add(write.id);
+			}
+		}
+		return writes.map((write) => {
+			if (write.type !== "document.copy") return write;
+			try {
+				if (changedDocumentIds.has(write.source.id)) {
+					throw new Error(`Fork source document ${write.source.id} is changed in the copy batch`);
+				}
+				const stored = this.materializeDocument(write.source.id, write.source.at);
+				if (stored === undefined) throw new Error(`Fork source document ${write.source.id} cannot be read`);
+				if (
+					stored.record.scope.kind !== "conversation" ||
+					write.record.scope.kind !== "conversation" ||
+					stored.record.kind !== write.record.kind ||
+					stored.record.key !== write.record.key ||
+					stored.record.history !== write.record.history ||
+					stored.record.fork !== write.record.fork
+				) {
+					throw new Error(`Fork source document ${write.source.id} does not match the copied record`);
+				}
+				return {
+					type: "document.create",
+					record: write.record,
+					content: { kind: "base", version: stored.version, value: stored.value },
+				};
+			} catch (error) {
+				throw new Error(`Document copy ${write.record.id} was rejected`, { cause: error });
+			}
+		});
+	}
+
+	private applyPreparedCommit(
+		prepared: readonly StorageWrite[],
+		documentActions: ReadonlyMap<DocumentId, DocumentAction>,
+		seq: Seq,
+	): Seq {
+		for (const write of prepared) {
+			switch (write.type) {
+				case "conversation": {
+					this.state.recordTypes.set(write.value.id, "conversation");
+					this.state.conversations.set(write.value.id, write.value);
+					insertSorted(this.state.conversationIds, write.value.id);
+					const owner = write.value.owner;
+					if (owner !== undefined) {
+						insertMapId(this.state.conversationIdsByOwnerConversation, owner.conversationId, write.value.id);
+						insertMapId(this.state.conversationIdsByOwnerTask, owner.taskId, write.value.id);
+					}
+					this.nextId = Math.max(this.nextId, write.value.id + 1);
+					break;
+				}
+				case "entry": {
+					this.state.recordTypes.set(write.value.id, "entry");
+					this.state.entries.set(write.value.id, write.value);
+					this.state.entryCommitSeqs.set(write.value.id, seq);
+					let ids = this.state.entryIds.get(write.value.conversationId);
+					if (ids === undefined) {
+						ids = [];
+						this.state.entryIds.set(write.value.conversationId, ids);
+					}
+					insertSorted(ids, write.value.id);
+					if (write.value.head !== undefined) {
+						let headIds = this.state.headEntryIds.get(write.value.conversationId);
+						if (headIds === undefined) {
+							headIds = [];
+							this.state.headEntryIds.set(write.value.conversationId, headIds);
+						}
+						insertSorted(headIds, write.value.id);
+					}
+					this.nextId = Math.max(this.nextId, write.value.id + 1);
+					break;
+				}
+				case "task": {
+					this.state.recordTypes.set(write.value.id, "task");
+					const previous = this.state.tasks.get(write.value.id);
+					if (previous === undefined) {
+						insertSorted(this.state.taskIds, write.value.id);
+						insertSorted(this.state.taskIdsByStatus[write.value.state.status], write.value.id);
+					} else if (previous.state.status !== write.value.state.status) {
+						removeSorted(this.state.taskIdsByStatus[previous.state.status], write.value.id);
+						insertSorted(this.state.taskIdsByStatus[write.value.state.status], write.value.id);
+					}
+					this.state.tasks.set(write.value.id, write.value);
+					this.nextId = Math.max(this.nextId, write.value.id + 1);
+					break;
+				}
+				case "submission": {
+					this.state.recordTypes.set(write.value.id, "submission");
+					const previous = this.state.submissions.get(write.value.id);
+					if (previous === undefined) {
+						insertSorted(this.state.submissionIds, write.value.id);
+						insertSorted(this.state.submissionIdsByStatus[write.value.status], write.value.id);
+					} else if (previous.status !== write.value.status) {
+						removeSorted(this.state.submissionIdsByStatus[previous.status], write.value.id);
+						insertSorted(this.state.submissionIdsByStatus[write.value.status], write.value.id);
+					}
+					if (previous?.requestId !== undefined) {
+						const previousRequests = this.state.submissionIdsByRequest.get(previous.conversationId);
+						if (previousRequests?.get(previous.requestId) === write.value.id) {
+							previousRequests.delete(previous.requestId);
+							if (previousRequests.size === 0) this.state.submissionIdsByRequest.delete(previous.conversationId);
+						}
+					}
+					this.state.submissions.set(write.value.id, write.value);
+					if (write.value.requestId !== undefined) {
+						let requests = this.state.submissionIdsByRequest.get(write.value.conversationId);
+						if (requests === undefined) {
+							requests = new Map();
+							this.state.submissionIdsByRequest.set(write.value.conversationId, requests);
+						}
+						requests.set(write.value.requestId, write.value.id);
+					}
+					this.nextId = Math.max(this.nextId, write.value.id + 1);
+					break;
+				}
+				case "document.copy":
+					throw new Error("Prepared document copy was not resolved");
+				case "document.create":
+				case "document.change":
+				case "document.retire":
+					break;
+			}
+		}
+
+		this.applyDocumentActions(documentActions, seq);
+		this.nextSeq = seq + 1;
+		return seq;
+	}
+
+	async mintId<I extends Id<string>>(): Promise<I> {
+		this.assertOpen();
+		if (!Number.isSafeInteger(this.nextId)) throw new Error("ID space is exhausted");
+		return idFromNumber<I>(this.nextId++);
+	}
+
+	async conversation(id: ConversationId, _context: Context): Promise<ConversationRecord | undefined> {
+		this.assertOpen();
+		const value = this.state.conversations.get(id);
+		return value === undefined ? undefined : clone(value);
+	}
+
+	async scanConversations(
+		query: ConversationQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<ConversationRecord, Cursor>> {
+		this.assertOpen();
+		const ids =
+			query.ownerTaskId !== undefined
+				? (this.state.conversationIdsByOwnerTask.get(query.ownerTaskId) ?? [])
+				: query.ownerConversationId !== undefined
+					? (this.state.conversationIdsByOwnerConversation.get(query.ownerConversationId) ?? [])
+					: this.state.conversationIds;
+		const start = scanStart(query.order, cursor, "ascending");
+		const values: ConversationRecord[] = [];
+		for (const index of scanIndexes(ids, start)) {
+			if (values.length > limit) break;
+			const value = this.state.conversations.get(ids[index]!)!;
+			if (query.ownerConversationId !== undefined && value.owner?.conversationId !== query.ownerConversationId) {
+				continue;
+			}
+			values.push(value);
+		}
+		return page(values, limit, start.order);
+	}
+
+	entry(id: EntryId, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
+	entry(
+		conversationId: ConversationId,
+		id: EntryId,
+		context: Context,
+	): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
+	async entry(
+		idOrConversationId: EntryId | ConversationId,
+		idOrContext: EntryId | Context,
+		context?: Context,
+	): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined> {
+		this.assertOpen();
+		if (context === undefined) {
+			const id = idFromNumber<EntryId>(idOrConversationId);
+			const entry = this.state.entries.get(id);
+			if (entry === undefined) return undefined;
+			return { entry: clone(entry), commitSeq: this.state.entryCommitSeqs.get(id)! };
+		}
+		if (typeof idOrContext !== "number") throw new StorageRequestError("Storage.entry() requires an entry ID");
+		const conversationId = idFromNumber<ConversationId>(idOrConversationId);
+		const id = idFromNumber<EntryId>(idOrContext);
+		const entry = this.visibleEntries(conversationId, id, id).next().value;
+		if (entry === undefined) return undefined;
+		return { entry: clone(entry), commitSeq: this.state.entryCommitSeqs.get(id)! };
+	}
+
+	async findLatestHeadMarker(
+		conversationId: ConversationId,
+		atOrBeforeEntryId: EntryId | undefined,
+		_context: Context,
+	): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> {
+		this.assertOpen();
+		if (!this.state.conversations.has(conversationId)) {
+			throw new StorageRequestError(`Unknown conversation: ${conversationId}`);
+		}
+		let currentId = conversationId;
+		let upperEntryId = atOrBeforeEntryId ?? Number.POSITIVE_INFINITY;
+		while (true) {
+			const ids = this.state.headEntryIds.get(currentId) ?? [];
+			const index = upperBound(ids, upperEntryId) - 1;
+			if (index >= 0) {
+				const entry = this.state.entries.get(ids[index])!;
+				return clone({ ...entry, head: entry.head! });
+			}
+			const conversation = this.state.conversations.get(currentId)!;
+			if (conversation.parent === undefined) return undefined;
+			upperEntryId = Math.min(upperEntryId, conversation.parent.at);
+			currentId = conversation.parent.conversationId;
+		}
+	}
+
+	async scanEntries(
+		query: EntryQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<EntryRecord, Cursor>> {
+		this.assertOpen();
+		const { order, after } = scanStart(query.order, cursor, "descending");
+		// The cursor narrows the bound on the side the scan moves away from.
+		let minEntryId: number | undefined = query.minEntryId;
+		let maxEntryId: number | undefined = query.maxEntryId;
+		if (after !== undefined && order === "descending") maxEntryId = Math.min(maxEntryId ?? after, after - 1);
+		if (after !== undefined && order === "ascending") minEntryId = Math.max(minEntryId ?? after, after + 1);
+		const visible: EntryRecord[] = [];
+		const entries =
+			order === "descending"
+				? this.visibleEntries(query.conversationId, minEntryId, maxEntryId)
+				: this.visibleEntriesAscending(query.conversationId, minEntryId, maxEntryId);
+		for (const entry of entries) {
+			visible.push(entry);
+			if (visible.length > limit) break;
+		}
+		return page(visible, limit, order);
+	}
+
+	async task(id: TaskId, _context: Context): Promise<StoredTask | undefined> {
+		this.assertOpen();
+		const value = this.state.tasks.get(id);
+		return value === undefined ? undefined : clone(value);
+	}
+
+	async scanTasks(
+		query: TaskQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<StoredTask, Cursor>> {
+		this.assertOpen();
+		const start = scanStart(query.order, cursor, "ascending");
+		const ids = query.status === undefined ? this.state.taskIds : this.state.taskIdsByStatus[query.status];
+		const values: StoredTask[] = [];
+		for (const index of scanIndexes(ids, start)) {
+			if (values.length > limit) break;
+			const value = this.state.tasks.get(ids[index]!)!;
+			if (query.conversationId !== undefined && value.conversationId !== query.conversationId) continue;
+			if (query.kind !== undefined && value.kind !== query.kind) continue;
+			if (query.abortRequested !== undefined && value.abortRequested !== query.abortRequested) continue;
+			if (query.background !== undefined && value.background !== query.background) continue;
+			values.push(value);
+		}
+		return page(values, limit, start.order);
+	}
+
+	async submission(id: SubmissionId, _context: Context): Promise<SubmissionRecord | undefined> {
+		this.assertOpen();
+		const value = this.state.submissions.get(id);
+		return value === undefined ? undefined : clone(value);
+	}
+
+	async scanSubmissions(
+		query: SubmissionQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<SubmissionRecord, Cursor>> {
+		this.assertOpen();
+		const start = scanStart(query.order, cursor, "ascending");
+		const ids =
+			query.status === undefined ? this.state.submissionIds : this.state.submissionIdsByStatus[query.status];
+		const values: SubmissionRecord[] = [];
+		for (const index of scanIndexes(ids, start)) {
+			if (values.length > limit) break;
+			const value = this.state.submissions.get(ids[index]!)!;
+			if (query.conversationId !== undefined && value.conversationId !== query.conversationId) continue;
+			values.push(value);
+		}
+		return page(values, limit, start.order);
+	}
+
+	async submissionByRequest(
+		conversationId: ConversationId,
+		requestId: string,
+		_context: Context,
+	): Promise<SubmissionRecord | undefined> {
+		this.assertOpen();
+		const id = this.state.submissionIdsByRequest.get(conversationId)?.get(requestId);
+		if (id === undefined) return undefined;
+		return clone(this.state.submissions.get(id)!);
+	}
+
+	async findDocument(
+		address: DocumentAddress,
+		at: DocumentPoint,
+		_context: Context,
+	): Promise<DocumentRecord | undefined> {
+		this.assertOpen();
+		const index = this.state.documentAddresses.get(addressKey(address));
+		if (at === "current") {
+			return index?.currentId === undefined ? undefined : clone(this.state.documents.get(index.currentId)!.record);
+		}
+		for (const id of index?.ids ?? []) {
+			const record = this.state.documents.get(id)!.record;
+			if (isAliveAt(record, at)) return clone(record);
+		}
+		return undefined;
+	}
+
+	async document(id: DocumentId, at: DocumentPoint, _context: Context): Promise<StoredDocument | undefined> {
+		this.assertOpen();
+		const stored = this.materializeDocument(id, at);
+		return stored === undefined
+			? undefined
+			: {
+					record: clone(stored.record),
+					version: stored.version,
+					value: clone(stored.value),
+					deltasSinceBase: stored.deltasSinceBase,
+				};
+	}
+
+	async scanDocuments(
+		query: DocumentQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<DocumentRecord, Cursor>> {
+		this.assertOpen();
+		const ids = this.state.documentIdsByScope.get(scopeKey(query.scope)) ?? [];
+		const after = cursorId(cursor);
+		const start = after === undefined ? 0 : upperBound(ids, after);
+		const values: DocumentRecord[] = [];
+		for (let index = start; index < ids.length && values.length <= limit; index++) {
+			const record = this.state.documents.get(ids[index]!)!.record;
+			if (query.kind !== undefined && record.kind !== query.kind) continue;
+			if (isAliveAt(record, query.at)) values.push(record);
+		}
+		return page(values, limit);
+	}
+
+	async close(_context: Context): Promise<void> {
+		this.closed = true;
+	}
+
+	private materializeDocument(id: DocumentId, at: DocumentPoint): StoredDocument | undefined {
+		const stored = this.state.documents.get(id);
+		if (stored === undefined) return undefined;
+		if (at !== "current" && isCurrentOnly(stored.record)) {
+			throw new StorageRequestError(`Document ${id} does not retain historical content`);
+		}
+		if (!isAliveAt(stored.record, at)) return undefined;
+		const revisions = at === "current" ? stored.revisions : stored.revisions.filter((revision) => revision.seq <= at);
+		let baseIndex = revisions.length - 1;
+		while (baseIndex >= 0 && revisions[baseIndex]!.kind !== "base") baseIndex--;
+		const base = revisions[baseIndex];
+		if (base?.kind !== "base") throw new Error(`Document ${id} is missing a required base`);
+		const value = applyImmutableBatches(
+			base.value,
+			documentDeltaBatches(id, base.version, revisions, baseIndex + 1),
+		) as JsonObject;
+		return { record: stored.record, version: base.version, value, deltasSinceBase: revisions.length - baseIndex - 1 };
+	}
+
+	private *visibleEntries(
+		conversationId: ConversationId,
+		minEntryId: number = Number.NEGATIVE_INFINITY,
+		maxEntryId: number = Number.POSITIVE_INFINITY,
+	): Generator<EntryRecord> {
+		if (!this.state.conversations.has(conversationId)) {
+			throw new StorageRequestError(`Unknown conversation: ${conversationId}`);
+		}
+		let currentId = conversationId;
+		let upperEntryId = maxEntryId;
+		while (true) {
+			const ids = this.state.entryIds.get(currentId) ?? [];
+			for (let index = upperBound(ids, upperEntryId) - 1; index >= 0; index--) {
+				const id = ids[index];
+				if (id < minEntryId) break;
+				yield this.state.entries.get(id)!;
+			}
+			const conversation = this.state.conversations.get(currentId)!;
+			if (conversation.parent === undefined) break;
+			upperEntryId = Math.min(upperEntryId, conversation.parent.at);
+			if (upperEntryId < minEntryId) break;
+			currentId = conversation.parent.conversationId;
+		}
+	}
+
+	/** Visible entries oldest first: the fork chain's segments from the root conversation forward. */
+	private *visibleEntriesAscending(
+		conversationId: ConversationId,
+		minEntryId: number = Number.NEGATIVE_INFINITY,
+		maxEntryId: number = Number.POSITIVE_INFINITY,
+	): Generator<EntryRecord> {
+		if (!this.state.conversations.has(conversationId)) {
+			throw new StorageRequestError(`Unknown conversation: ${conversationId}`);
+		}
+		const segments: { readonly conversationId: ConversationId; readonly upper: number }[] = [];
+		let currentId = conversationId;
+		let upperEntryId = maxEntryId;
+		while (true) {
+			segments.push({ conversationId: currentId, upper: upperEntryId });
+			const conversation = this.state.conversations.get(currentId)!;
+			if (conversation.parent === undefined) break;
+			upperEntryId = Math.min(upperEntryId, conversation.parent.at);
+			if (upperEntryId < minEntryId) break;
+			currentId = conversation.parent.conversationId;
+		}
+		for (const segment of segments.reverse()) {
+			const ids = this.state.entryIds.get(segment.conversationId) ?? [];
+			for (let index = lowerBound(ids, minEntryId); index < ids.length; index++) {
+				const id = ids[index]!;
+				if (id > segment.upper) break;
+				yield this.state.entries.get(id)!;
+			}
+		}
+	}
+
+	private checkGlobalIds(writes: readonly StorageWrite[]): void {
+		const claimed = new Map<Id<string>, TableName>();
+		for (const write of writes) {
+			if (write.type === "document.change" || write.type === "document.retire") continue;
+			const document = write.type === "document.create" || write.type === "document.copy";
+			const table: TableName = document ? "document" : write.type;
+			const id = document ? write.record.id : write.value.id;
+			const existing = tableContaining(this.state, id);
+			const earlier = claimed.get(id);
+			if (table === "conversation" || table === "entry" || table === "document") {
+				if (existing !== undefined) throw new Error(`ID ${id} already belongs to ${existing}`);
+				if (earlier !== undefined) throw new Error(`ID ${id} is written more than once`);
+			} else {
+				if (existing !== undefined && existing !== table) {
+					throw new Error(`ID ${id} already belongs to ${existing}`);
+				}
+				if (earlier !== undefined && earlier !== table) throw new Error(`ID ${id} is written as two record types`);
+			}
+			claimed.set(id, table);
+		}
+	}
+
+	private prepareDocumentActions(writes: readonly StorageWrite[]): Map<DocumentId, DocumentAction> {
+		const actions = new Map<DocumentId, DocumentAction>();
+		for (const write of writes) {
+			if (write.type !== "document.create" && write.type !== "document.change" && write.type !== "document.retire") {
+				continue;
+			}
+			const id = write.type === "document.create" ? write.record.id : write.id;
+			let action = actions.get(id);
+			if (action === undefined) {
+				action = { retire: false };
+				actions.set(id, action);
+			}
+			switch (write.type) {
+				case "document.create":
+					if (action.create !== undefined || action.content !== undefined) {
+						throw new Error(`Document ${id} has more than one content command`);
+					}
+					action.create = write.record;
+					action.content = write.content;
+					break;
+				case "document.change":
+					if (action.content !== undefined) throw new Error(`Document ${id} has more than one content command`);
+					action.content = write.content;
+					break;
+				case "document.retire":
+					if (action.retire) throw new Error(`Document ${id} is retired more than once`);
+					action.retire = true;
+					break;
+			}
+		}
+		return actions;
+	}
+
+	private checkDocumentActions(actions: ReadonlyMap<DocumentId, DocumentAction>): void {
+		const liveCounts = new Map<string, number>();
+		for (const [id, action] of actions) {
+			const existing = this.state.documents.get(id);
+			if (action.create === undefined && existing === undefined) throw new Error(`Unknown document: ${id}`);
+			if (action.create !== undefined && existing !== undefined) throw new Error(`Document ${id} already exists`);
+			if (existing?.record.retiredAt !== undefined) throw new Error(`Document ${id} is retired`);
+			const previous = existing?.revisions.at(-1);
+			if (action.content?.kind === "delta") {
+				if (previous === undefined) throw new Error(`Document ${id} delta has no base`);
+				if (previous.version !== action.content.version) {
+					throw new Error(`Document ${id} version transition requires a base`);
+				}
+			}
+
+			const key = action.create === undefined ? recordAddressKey(existing!.record) : recordAddressKey(action.create);
+			const currentId = this.state.documentAddresses.get(key)?.currentId;
+			let live = liveCounts.get(key);
+			if (live === undefined) live = currentId === undefined ? 0 : 1;
+			if (action.retire && currentId === id) live--;
+			if (action.create !== undefined && !action.retire) live++;
+			liveCounts.set(key, live);
+		}
+
+		for (const live of liveCounts.values()) {
+			if (live > 1) throw new Error(`Document address already has a current incarnation`);
+		}
+	}
+
+	private applyDocumentActions(actions: ReadonlyMap<DocumentId, DocumentAction>, seq: Seq): void {
+		for (const [id, action] of actions) {
+			let stored = this.state.documents.get(id);
+			if (action.create !== undefined) {
+				const record: DocumentRecord = {
+					...action.create,
+					createdAt: seq,
+					...(action.retire ? { retiredAt: seq } : {}),
+				};
+				stored = { record, revisions: [{ ...action.content!, seq }] };
+				this.state.recordTypes.set(id, "document");
+				this.state.documents.set(id, stored);
+
+				const key = recordAddressKey(record);
+				let address = this.state.documentAddresses.get(key);
+				if (address === undefined) {
+					address = { ids: [] };
+					this.state.documentAddresses.set(key, address);
+				}
+				insertSorted(address.ids, id);
+
+				let scopeIds = this.state.documentIdsByScope.get(scopeKey(record.scope));
+				if (scopeIds === undefined) {
+					scopeIds = [];
+					this.state.documentIdsByScope.set(scopeKey(record.scope), scopeIds);
+				}
+				insertSorted(scopeIds, id);
+				this.nextId = Math.max(this.nextId, id + 1);
+			} else if (action.content !== undefined) {
+				const revision = { ...action.content, seq } as DocumentRevision;
+				if (revision.kind === "base" && isCurrentOnly(stored!.record)) stored!.revisions = [revision];
+				else stored!.revisions.push(revision);
+			}
+
+			if (action.retire && action.create === undefined) stored!.record = { ...stored!.record, retiredAt: seq };
+			if (action.retire && isCurrentOnly(stored!.record)) stored!.revisions = [];
+			if (action.create !== undefined || action.retire) {
+				const address = this.state.documentAddresses.get(recordAddressKey(stored!.record))!;
+				if (action.retire && address.currentId === id) delete address.currentId;
+				if (action.create !== undefined && !action.retire) address.currentId = id;
+			}
+		}
+	}
+
+	private assertOpen(): void {
+		if (this.closed) throw new Error("MemoryStorage is closed");
+	}
+}

third_party/pi/packages/durable/src/storage/scan.ts

@@ -0,0 +1,40 @@
+import { StorageRequestError } from "../errors.ts";
+import type { Id, ScanOrder } from "../types.ts";
+
+/** Where a built-in storage scan starts: its order and the last ID a previous page returned. */
+export type ScanStart = {
+	readonly order: ScanOrder;
+	readonly after: number | undefined;
+};
+
+/**
+ * Resolve a scan's order and position. A cursor continues in the order it was created with, whether the query repeats
+ * that order or omits it; a different `order` throws. A cursor without an order, written before scans had one,
+ * continues in the scan's default order.
+ */
+export function scanStart(
+	requested: ScanOrder | undefined,
+	cursor: Readonly<Record<string, unknown>> | undefined,
+	fallback: ScanOrder,
+): ScanStart {
+	if (requested !== undefined && requested !== "ascending" && requested !== "descending") {
+		throw new StorageRequestError(`Invalid scan order: ${String(requested)}`);
+	}
+	if (cursor === undefined) return { order: requested ?? fallback, after: undefined };
+	const { after, order: stored } = cursor;
+	if (typeof after !== "number" || !Number.isSafeInteger(after))
+		throw new StorageRequestError("Invalid storage cursor");
+	if (stored !== undefined && stored !== "ascending" && stored !== "descending") {
+		throw new StorageRequestError("Invalid storage cursor");
+	}
+	const order = stored ?? fallback;
+	if (requested !== undefined && requested !== order) {
+		throw new StorageRequestError(`The cursor continues a ${order} scan; the query asks for ${requested}`);
+	}
+	return { order, after };
+}
+
+/** The continuation of a scan whose last returned item has `id`. */
+export function nextCursor(id: Id<string>, order: ScanOrder): { readonly after: number; readonly order: ScanOrder } {
+	return { after: id, order };
+}

third_party/pi/packages/durable/src/storage/sqlite/cloudflare.ts

@@ -0,0 +1,192 @@
+import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "./database.ts";
+import { SqliteStorage, type SqliteStorageOptions } from "./storage.ts";
+
+/** Values a Durable Object SQL binding accepts and returns. */
+type DurableObjectSqlValue = ArrayBuffer | string | number | null;
+
+/**
+ * The parts of a SQLite-backed Durable Object's `ctx.storage` this adapter uses, typed structurally so the package
+ * needs no Workers type dependency.
+ */
+export type DurableObjectSqliteStorage = {
+	readonly sql: {
+		exec(
+			query: string,
+			...bindings: DurableObjectSqlValue[]
+		): {
+			next():
+				| { readonly done?: false; readonly value: Record<string, DurableObjectSqlValue> }
+				| { readonly done: true; readonly value?: undefined };
+			toArray(): Record<string, DurableObjectSqlValue>[];
+		};
+	};
+	transaction<T>(closure: () => Promise<T>): Promise<T>;
+};
+
+/** Runs synchronous operations immediately when idle and holds queued calls behind a transaction. */
+class SerialQueue {
+	private tail: Promise<void> = Promise.resolve();
+	private pending = 0;
+
+	run<T>(operation: () => T | Promise<T>): Promise<T> {
+		if (this.pending > 0) return this.enqueue(operation);
+		try {
+			return Promise.resolve(operation());
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+	hold<T>(operation: () => Promise<T>): Promise<T> {
+		return this.enqueue(operation);
+	}
+
+	private enqueue<T>(operation: () => T | Promise<T>): Promise<T> {
+		this.pending++;
+		const result = this.tail.then(operation).finally(() => {
+			this.pending--;
+		});
+		this.tail = result.then(
+			() => {},
+			() => {},
+		);
+		return result;
+	}
+}
+
+const storageQueues = new WeakMap<DurableObjectSqliteStorage, SerialQueue>();
+
+function queueFor(storage: DurableObjectSqliteStorage): SerialQueue {
+	let queue = storageQueues.get(storage);
+	if (queue === undefined) {
+		queue = new SerialQueue();
+		storageQueues.set(storage, queue);
+	}
+	return queue;
+}
+
+/** Durable Object SQL binds numbers as doubles: a `bigint` outside the safe integer range would lose precision. */
+function bindValue(value: SqliteValue): DurableObjectSqlValue {
+	if (value instanceof Uint8Array) return value.slice().buffer as ArrayBuffer;
+	if (typeof value !== "bigint") return value;
+	if (value < BigInt(Number.MIN_SAFE_INTEGER) || value > BigInt(Number.MAX_SAFE_INTEGER)) {
+		throw new RangeError(`Durable Object SQL cannot bind ${value} without losing precision`);
+	}
+	return Number(value);
+}
+
+const bind = (params: readonly SqliteValue[]): DurableObjectSqlValue[] => params.map(bindValue);
+
+/** A row as returned, with `BLOB` columns converted to `Uint8Array` in place; no other copy is made. */
+function row<T>(row: Record<string, DurableObjectSqlValue>): T {
+	for (const key in row) {
+		const value = row[key];
+		if (value instanceof ArrayBuffer) (row as Record<string, unknown>)[key] = new Uint8Array(value);
+	}
+	return row as T;
+}
+
+/** Executes SQL through `ctx.storage.sql`, which has no separate prepare step. */
+class DurableObjectSqliteExecutor implements SqliteExecutor {
+	protected readonly storage: DurableObjectSqliteStorage;
+
+	constructor(storage: DurableObjectSqliteStorage) {
+		this.storage = storage;
+	}
+
+	async exec(sql: string): Promise<void> {
+		this.check();
+		this.storage.sql.exec(sql);
+	}
+
+	async run(sql: string, ...params: SqliteValue[]): Promise<void> {
+		this.check();
+		this.storage.sql.exec(sql, ...bind(params));
+	}
+
+	async get<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T | undefined> {
+		this.check();
+		const result = this.storage.sql.exec(sql, ...bind(params)).next();
+		return result.done ? undefined : row<T>(result.value);
+	}
+
+	async all<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T[]> {
+		this.check();
+		return this.storage.sql
+			.exec(sql, ...bind(params))
+			.toArray()
+			.map((value) => row<T>(value));
+	}
+
+	/** Throws when this handle may no longer run SQL. */
+	protected check(): void {}
+}
+
+/** The handle a transaction callback receives; invalid once the callback settles. */
+class DurableObjectSqliteTransaction extends DurableObjectSqliteExecutor {
+	active = true;
+
+	protected override check(): void {
+		if (!this.active) throw new Error("SQLite transaction handle is no longer active");
+	}
+}
+
+/**
+ * `SqliteDatabase` adapter for the SQLite storage of a Cloudflare Durable Object. Adapters given the same storage
+ * object share a queue; direct calls to the underlying storage bypass it. Integers are JavaScript numbers; binding a
+ * `bigint` outside the safe integer range throws.
+ */
+export class DurableObjectSqliteDatabase extends DurableObjectSqliteExecutor implements SqliteDatabase {
+	private readonly queue: SerialQueue;
+
+	constructor(storage: DurableObjectSqliteStorage) {
+		super(storage);
+		this.queue = queueFor(storage);
+	}
+
+	override exec(sql: string): Promise<void> {
+		return this.queue.run(() => super.exec(sql));
+	}
+
+	override run(sql: string, ...params: SqliteValue[]): Promise<void> {
+		return this.queue.run(() => super.run(sql, ...params));
+	}
+
+	override get<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T | undefined> {
+		return this.queue.run(() => super.get<T>(sql, ...params));
+	}
+
+	override all<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T[]> {
+		return this.queue.run(() => super.all<T>(sql, ...params));
+	}
+
+	/** `ctx.storage.transaction()` commits when the callback resolves and rolls back when it rejects. */
+	transaction<T>(callback: (transaction: SqliteExecutor) => Promise<T>): Promise<T> {
+		return this.queue.hold(() =>
+			this.storage.transaction(async () => {
+				const transaction = new DurableObjectSqliteTransaction(this.storage);
+				try {
+					return await callback(transaction);
+				} finally {
+					transaction.active = false;
+				}
+			}),
+		);
+	}
+
+	/** Waits for queued work. The Durable Object owns its storage, so nothing else closes. */
+	close(): Promise<void> {
+		return this.queue.run(() => {});
+	}
+}
+
+/**
+ * Open durable storage on a SQLite-backed Durable Object, optionally under a table prefix. Calls given the same
+ * storage object share an operation queue.
+ */
+export function openDurableObjectSqliteStorage(
+	storage: DurableObjectSqliteStorage,
+	options: SqliteStorageOptions = {},
+): Promise<SqliteStorage> {
+	return SqliteStorage.open(new DurableObjectSqliteDatabase(storage), options);
+}

third_party/pi/packages/durable/src/storage/sqlite/database.ts

@@ -0,0 +1,38 @@
+/** Values supported by the portable SQLite storage core. */
+export type SqliteValue = null | number | bigint | string | Uint8Array;
+
+/**
+ * Asynchronous SQL operations shared by a database and its transaction handles.
+ *
+ * `exec` runs SQL text without bindings and may contain several statements. `run`, `get`, and `all`
+ * execute one statement with positional bindings. Adapters may cache prepared statements by SQL text,
+ * so callers pass values as bindings instead of interpolating them.
+ */
+export interface SqliteExecutor {
+	exec(sql: string): Promise<void>;
+	run(sql: string, ...params: SqliteValue[]): Promise<void>;
+	get<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T | undefined>;
+	all<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T[]>;
+}
+
+/**
+ * Minimal database facade required by `SqliteStorage`.
+ *
+ * All operations are asynchronous so adapters may execute outside the harness runtime.
+ *
+ * `transaction` passes the callback a transaction handle. All work in the transaction
+ * must use that handle; the handle is invalid after the callback settles. Adapters must
+ * queue unrelated operations and other transactions until the transaction finishes. The
+ * returned promise settles after commit or rollback. Calling the database itself (including
+ * `transaction` or `close`) from inside a callback therefore waits for that transaction and
+ * never settles.
+ *
+ * When the callback rejects, the adapter must roll the transaction back before rejecting
+ * with that same error. If rollback fails, it must reject with a different error (for
+ * example an `AggregateError`) so callers cannot mistake the callback error for a
+ * guaranteed rollback.
+ */
+export interface SqliteDatabase extends SqliteExecutor {
+	transaction<T>(callback: (transaction: SqliteExecutor) => Promise<T>): Promise<T>;
+	close(): Promise<void>;
+}

third_party/pi/packages/durable/src/storage/sqlite/index.ts

@@ -0,0 +1,8 @@
+export type { SqliteDatabase, SqliteExecutor, SqliteValue } from "./database.ts";
+export {
+	applySqliteMigrations,
+	CURRENT_SQLITE_SCHEMA_VERSION,
+	SQLITE_MIGRATIONS,
+	type SqliteMigration,
+} from "./migrations.ts";
+export { SqliteStorage, type SqliteStorageOptions } from "./storage.ts";

third_party/pi/packages/durable/src/storage/sqlite/migrations.ts

@@ -0,0 +1,140 @@
+import type { SqliteDatabase } from "./database.ts";
+import { DEFAULT_SQLITE_SCHEMA, type SqliteSchema } from "./schema.ts";
+
+export type SqliteMigration = {
+	readonly version: number;
+	readonly statements: readonly string[];
+};
+
+/** Build immutable, ordered schema history for a set of SQLite identifiers. */
+export function createSqliteMigrations(schema: SqliteSchema): readonly SqliteMigration[] {
+	const { tables, indexes } = schema;
+	// next_id is TEXT because node:sqlite rejects INTEGER results outside JavaScript's safe integer range.
+	const initialSchema: readonly string[] = [
+		`CREATE TABLE ${tables.durableMetadata} (
+			singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
+			next_id TEXT NOT NULL,
+			next_seq INTEGER NOT NULL
+		) STRICT`,
+		`INSERT INTO ${tables.durableMetadata} (singleton, next_id, next_seq) VALUES (1, '2', 1)`,
+		`CREATE TABLE ${tables.recordIds} (
+			id INTEGER PRIMARY KEY,
+			record_type TEXT NOT NULL CHECK (record_type IN ('conversation', 'entry', 'task', 'submission', 'document'))
+		) STRICT`,
+		`CREATE TABLE ${tables.conversations} (
+			id INTEGER PRIMARY KEY,
+			owner_conversation_id INTEGER,
+			owner_task_id INTEGER,
+			record TEXT NOT NULL CHECK (json_valid(record))
+		) STRICT`,
+		`CREATE INDEX ${indexes.conversationsByOwnerConversation} ON ${tables.conversations} (owner_conversation_id, id)`,
+		`CREATE INDEX ${indexes.conversationsByOwnerTask} ON ${tables.conversations} (owner_task_id, id)`,
+		`CREATE TABLE ${tables.entries} (
+			id INTEGER PRIMARY KEY,
+			conversation_id INTEGER NOT NULL,
+			head INTEGER,
+			commit_seq INTEGER NOT NULL,
+			record TEXT NOT NULL CHECK (json_valid(record))
+		) STRICT`,
+		`CREATE INDEX ${indexes.entriesByConversation} ON ${tables.entries} (conversation_id, id DESC)`,
+		`CREATE INDEX ${indexes.entryHeadsByConversation} ON ${tables.entries} (conversation_id, id DESC) WHERE head IS NOT NULL`,
+		`CREATE TABLE ${tables.tasks} (
+			id INTEGER PRIMARY KEY,
+			conversation_id INTEGER NOT NULL,
+			kind TEXT NOT NULL,
+			status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'waiting', 'completing', 'terminal')),
+			abort_requested INTEGER NOT NULL CHECK (abort_requested IN (0, 1)),
+			background INTEGER NOT NULL CHECK (background IN (0, 1)),
+			record TEXT NOT NULL CHECK (json_valid(record))
+		) STRICT`,
+		`CREATE INDEX ${indexes.tasksByStatus} ON ${tables.tasks} (status, id)`,
+		`CREATE INDEX ${indexes.tasksByConversation} ON ${tables.tasks} (conversation_id, id)`,
+		`CREATE INDEX ${indexes.tasksByKind} ON ${tables.tasks} (kind, id)`,
+		`CREATE INDEX ${indexes.tasksByAbortRequested} ON ${tables.tasks} (abort_requested, id)`,
+		`CREATE INDEX ${indexes.tasksByBackground} ON ${tables.tasks} (background, id)`,
+		`CREATE TABLE ${tables.submissions} (
+			id INTEGER PRIMARY KEY,
+			conversation_id INTEGER NOT NULL,
+			request_id TEXT,
+			status TEXT NOT NULL CHECK (status IN ('queued', 'placed', 'done', 'unanswered')),
+			record TEXT NOT NULL CHECK (json_valid(record))
+		) STRICT`,
+		`CREATE INDEX ${indexes.submissionsByRequest} ON ${tables.submissions} (conversation_id, request_id)`,
+		`CREATE INDEX ${indexes.submissionsByConversation} ON ${tables.submissions} (conversation_id, id)`,
+		`CREATE INDEX ${indexes.submissionsByStatus} ON ${tables.submissions} (status, id)`,
+		`CREATE TABLE ${tables.documents} (
+			id INTEGER PRIMARY KEY,
+			kind TEXT NOT NULL,
+			family INTEGER NOT NULL CHECK (family IN (0, 1)),
+			key_value TEXT NOT NULL,
+			scope_kind TEXT NOT NULL CHECK (scope_kind IN ('session', 'conversation', 'task')),
+			owner_id INTEGER NOT NULL,
+			created_at INTEGER NOT NULL,
+			retired_at INTEGER,
+			record TEXT NOT NULL CHECK (json_valid(record))
+		) STRICT`,
+		`CREATE INDEX ${indexes.documentsByAddress}
+			ON ${tables.documents} (kind, scope_kind, owner_id, family, key_value, created_at DESC, retired_at)`,
+		`CREATE INDEX ${indexes.documentsByScope} ON ${tables.documents} (scope_kind, owner_id, id)`,
+		`CREATE INDEX ${indexes.documentsByScopeKind} ON ${tables.documents} (scope_kind, owner_id, kind, id)`,
+		`CREATE TABLE ${tables.documentRevisions} (
+			document_id INTEGER NOT NULL,
+			seq INTEGER NOT NULL,
+			kind TEXT NOT NULL CHECK (kind IN ('base', 'delta')),
+			version INTEGER NOT NULL,
+			content TEXT NOT NULL CHECK (json_valid(content)),
+			PRIMARY KEY (document_id, seq)
+		) STRICT`,
+		`CREATE INDEX ${indexes.documentRevisionsByKind} ON ${tables.documentRevisions} (document_id, kind, seq DESC)`,
+	];
+	return [{ version: 1, statements: initialSchema }];
+}
+
+export const SQLITE_MIGRATIONS: readonly SqliteMigration[] = createSqliteMigrations(DEFAULT_SQLITE_SCHEMA);
+
+export const CURRENT_SQLITE_SCHEMA_VERSION = SQLITE_MIGRATIONS.at(-1)?.version ?? 0;
+
+type SchemaRow = { readonly version: number };
+
+/** Apply all pending schema migrations atomically. */
+export async function applySqliteMigrations(
+	database: SqliteDatabase,
+	migrations: readonly SqliteMigration[] = SQLITE_MIGRATIONS,
+): Promise<void> {
+	await applySqliteMigrationsForSchema(database, DEFAULT_SQLITE_SCHEMA, migrations);
+}
+
+/** Apply migrations rendered for a particular schema. */
+export async function applySqliteMigrationsForSchema(
+	database: SqliteDatabase,
+	schema: SqliteSchema,
+	migrations: readonly SqliteMigration[] = createSqliteMigrations(schema),
+): Promise<void> {
+	for (let index = 0; index < migrations.length; index++) {
+		if (migrations[index]?.version !== index + 1) {
+			throw new Error("Durable SQLite migrations must have contiguous versions starting at 1");
+		}
+	}
+
+	const { durableSchema } = schema.tables;
+	await database.transaction(async (transaction) => {
+		await transaction.exec(`CREATE TABLE IF NOT EXISTS ${durableSchema} (
+			singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
+			version INTEGER NOT NULL CHECK (version >= 0)
+		) STRICT`);
+		await transaction.run(`INSERT OR IGNORE INTO ${durableSchema} (singleton, version) VALUES (1, 0)`);
+		const row = await transaction.get<SchemaRow>(`SELECT version FROM ${durableSchema} WHERE singleton = 1`);
+		if (row === undefined) throw new Error("Durable SQLite schema metadata is missing");
+		const currentVersion = migrations.at(-1)?.version ?? 0;
+		if (row.version > currentVersion) {
+			throw new Error(
+				`Durable SQLite schema version ${row.version} is newer than supported version ${currentVersion}`,
+			);
+		}
+		for (const migration of migrations) {
+			if (migration.version <= row.version) continue;
+			for (const statement of migration.statements) await transaction.exec(statement);
+			await transaction.run(`UPDATE ${durableSchema} SET version = ? WHERE singleton = 1`, migration.version);
+		}
+	});
+}

third_party/pi/packages/durable/src/storage/sqlite/node.ts

@@ -0,0 +1,216 @@
+import { mkdir } from "node:fs/promises";
+import { dirname } from "node:path";
+import type { StatementSync } from "node:sqlite";
+import { DatabaseSync } from "node:sqlite";
+import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "./database.ts";
+import { validateSqliteSchemaPrefix } from "./schema.ts";
+import { SqliteStorage, type SqliteStorageOptions } from "./storage.ts";
+
+/** Node SQLite connection settings for a durable storage file. */
+export type NodeSqliteDatabaseOptions = {
+	/** SQLite WAL auto-checkpoint threshold. SQLite and this adapter default to 1,000 pages; 0 disables it. */
+	readonly walAutoCheckpointPages?: number;
+	/** Time SQLite waits for a competing file lock. SQLite defaults to 0; this adapter defaults to 5,000 ms. */
+	readonly busyTimeoutMs?: number;
+};
+
+/** Node SQLite connection settings plus durable storage options. */
+export type NodeSqliteStorageOptions = NodeSqliteDatabaseOptions & SqliteStorageOptions;
+
+const DEFAULT_WAL_AUTO_CHECKPOINT_PAGES = 1_000;
+const DEFAULT_BUSY_TIMEOUT_MS = 5_000;
+
+type TransactionScope = { active: boolean };
+
+const ignore = (): void => {};
+
+/**
+ * Runs operations in call order. An operation starts immediately when nothing is running or waiting;
+ * otherwise it waits for everything before it. An asynchronous operation holds the queue until it settles.
+ */
+class SerialOperationQueue {
+	private tail: Promise<void> = Promise.resolve();
+	private pending = 0;
+
+	run<T>(operation: () => T): Promise<T> {
+		if (this.pending > 0) return this.enqueue(operation);
+		try {
+			return Promise.resolve(operation());
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+	runAsync<T>(operation: () => Promise<T>): Promise<T> {
+		if (this.pending > 0) return this.enqueue(operation);
+		this.pending++;
+		// Publish the barrier before the operation starts, so calls it makes synchronously wait behind it.
+		const { promise: barrier, resolve: releaseBarrier } = Promise.withResolvers<void>();
+		this.tail = barrier;
+		let started: Promise<T>;
+		try {
+			started = operation();
+		} catch (error) {
+			started = Promise.reject(error);
+		}
+		return started.finally(() => {
+			this.pending--;
+			releaseBarrier();
+		});
+	}
+
+	private enqueue<T>(operation: () => T | Promise<T>): Promise<T> {
+		this.pending++;
+		return this.release(this.tail.then(operation));
+	}
+
+	private release<T>(operation: Promise<T>): Promise<T> {
+		const settled = operation.finally(() => {
+			this.pending--;
+		});
+		this.tail = settled.then(ignore, ignore);
+		return settled;
+	}
+}
+
+/**
+ * Executes SQL on one connection. Prepared statements are cached per connection by SQL text, so the
+ * database and its transaction handles share them across transactions.
+ */
+abstract class NodeSqliteExecutor implements SqliteExecutor {
+	protected readonly database: DatabaseSync;
+	protected readonly statements: Map<string, StatementSync>;
+
+	constructor(database: DatabaseSync, statements: Map<string, StatementSync>) {
+		this.database = database;
+		this.statements = statements;
+	}
+
+	exec(sql: string): Promise<void> {
+		return this.runOperation(() => {
+			this.database.exec(sql);
+		});
+	}
+
+	run(sql: string, ...params: SqliteValue[]): Promise<void> {
+		return this.runOperation(() => {
+			this.statement(sql).run(...params);
+		});
+	}
+
+	get<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T | undefined> {
+		return this.runOperation(() => this.statement(sql).get(...params) as T | undefined);
+	}
+
+	all<T extends object>(sql: string, ...params: SqliteValue[]): Promise<T[]> {
+		return this.runOperation(() => this.statement(sql).all(...params) as T[]);
+	}
+
+	protected abstract runOperation<T>(operation: () => T): Promise<T>;
+
+	private statement(sql: string): StatementSync {
+		let statement = this.statements.get(sql);
+		if (statement === undefined) {
+			statement = this.database.prepare(sql);
+			this.statements.set(sql, statement);
+		}
+		return statement;
+	}
+}
+
+class NodeSqliteTransaction extends NodeSqliteExecutor {
+	private readonly scope: TransactionScope;
+
+	constructor(database: DatabaseSync, statements: Map<string, StatementSync>, scope: TransactionScope) {
+		super(database, statements);
+		this.scope = scope;
+	}
+
+	protected async runOperation<T>(operation: () => T): Promise<T> {
+		if (!this.scope.active) throw new Error("SQLite transaction handle is no longer active");
+		return operation();
+	}
+}
+
+/** `SqliteDatabase` adapter backed by Node's built-in `node:sqlite`. */
+export class NodeSqliteDatabase extends NodeSqliteExecutor implements SqliteDatabase {
+	private readonly access = new SerialOperationQueue();
+	private closed = false;
+
+	constructor(database: DatabaseSync) {
+		super(database, new Map());
+	}
+
+	transaction<T>(callback: (transaction: SqliteExecutor) => Promise<T>): Promise<T> {
+		return this.access.runAsync(async () => {
+			this.database.exec("BEGIN IMMEDIATE");
+			const scope = { active: true };
+			try {
+				const result = await callback(new NodeSqliteTransaction(this.database, this.statements, scope));
+				scope.active = false;
+				this.database.exec("COMMIT");
+				return result;
+			} catch (error) {
+				scope.active = false;
+				try {
+					this.database.exec("ROLLBACK");
+				} catch (rollbackError) {
+					throw new AggregateError([error, rollbackError], "SQLite transaction failed and rollback failed");
+				}
+				throw error;
+			}
+		});
+	}
+
+	close(): Promise<void> {
+		return this.access.run(() => {
+			if (this.closed) return;
+			this.closed = true;
+			this.statements.clear();
+			try {
+				this.database.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+			} finally {
+				this.database.close();
+			}
+		});
+	}
+
+	protected runOperation<T>(operation: () => T): Promise<T> {
+		return this.access.run(operation);
+	}
+}
+
+/** Open and configure a Node-backed SQLite database facade. */
+export async function openNodeSqliteDatabase(
+	path: string,
+	options: NodeSqliteDatabaseOptions = {},
+): Promise<NodeSqliteDatabase> {
+	const checkpointPages = options.walAutoCheckpointPages ?? DEFAULT_WAL_AUTO_CHECKPOINT_PAGES;
+	const timeout = options.busyTimeoutMs ?? DEFAULT_BUSY_TIMEOUT_MS;
+	if (path !== ":memory:") await mkdir(dirname(path), { recursive: true });
+	const database = new DatabaseSync(path, { timeout });
+	const adapter = new NodeSqliteDatabase(database);
+	try {
+		await adapter.exec("PRAGMA journal_mode = WAL");
+		await adapter.exec("PRAGMA synchronous = NORMAL");
+		await adapter.exec(`PRAGMA wal_autocheckpoint = ${checkpointPages}`);
+		return adapter;
+	} catch (error) {
+		try {
+			await adapter.close();
+		} catch {
+			// Preserve the configuration failure.
+		}
+		throw error;
+	}
+}
+
+/** Open or create file-backed durable storage using Node's built-in SQLite, optionally under a table prefix. */
+export async function openNodeSqliteStorage(
+	path: string,
+	options: NodeSqliteStorageOptions = {},
+): Promise<SqliteStorage> {
+	const { prefix, ...databaseOptions } = options;
+	if (prefix !== undefined) validateSqliteSchemaPrefix(prefix);
+	return SqliteStorage.open(await openNodeSqliteDatabase(path, databaseOptions), { prefix });
+}

third_party/pi/packages/durable/src/storage/sqlite/schema.ts

@@ -0,0 +1,69 @@
+/**
+ * Immutable identifier history for migrations and runtime queries. Never change or remove an existing value; a
+ * rename adds a new entry so historical migrations keep rendering their original object names.
+ */
+const TABLE_NAMES = {
+	durableSchema: "durable_schema",
+	durableMetadata: "durable_metadata",
+	recordIds: "record_ids",
+	conversations: "conversations",
+	entries: "entries",
+	tasks: "tasks",
+	submissions: "submissions",
+	documents: "documents",
+	documentRevisions: "document_revisions",
+} as const;
+
+const INDEX_NAMES = {
+	conversationsByOwnerConversation: "conversations_by_owner_conversation",
+	conversationsByOwnerTask: "conversations_by_owner_task",
+	entriesByConversation: "entries_by_conversation",
+	entryHeadsByConversation: "entry_heads_by_conversation",
+	tasksByStatus: "tasks_by_status",
+	tasksByConversation: "tasks_by_conversation",
+	tasksByKind: "tasks_by_kind",
+	tasksByAbortRequested: "tasks_by_abort_requested",
+	tasksByBackground: "tasks_by_background",
+	submissionsByRequest: "submissions_by_request",
+	submissionsByConversation: "submissions_by_conversation",
+	submissionsByStatus: "submissions_by_status",
+	documentsByAddress: "documents_by_address",
+	documentsByScope: "documents_by_scope",
+	documentsByScopeKind: "documents_by_scope_kind",
+	documentRevisionsByKind: "document_revisions_by_kind",
+} as const;
+
+type SqliteIdentifiers<T extends Record<string, string>> = { readonly [K in keyof T]: string };
+
+export type SqliteSchema = {
+	readonly tables: SqliteIdentifiers<typeof TABLE_NAMES>;
+	readonly indexes: SqliteIdentifiers<typeof INDEX_NAMES>;
+};
+
+/** Validate a prefix before using it as part of a SQLite identifier. */
+export function validateSqliteSchemaPrefix(prefix: string): void {
+	if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(prefix)) {
+		throw new Error(`Invalid SQLite schema prefix ${JSON.stringify(prefix)}`);
+	}
+	if (prefix.toLowerCase().startsWith("sqlite_")) {
+		throw new Error(`SQLite schema prefix ${JSON.stringify(prefix)} is reserved`);
+	}
+}
+
+function prefixIdentifiers<T extends Record<string, string>>(names: T, prefix: string): SqliteIdentifiers<T> {
+	return Object.fromEntries(
+		Object.entries(names).map(([key, name]) => [key, `${prefix}${name}`]),
+	) as SqliteIdentifiers<T>;
+}
+
+/** Build the identifiers used by every migration and storage query. */
+export function createSqliteSchema(prefix?: string): SqliteSchema {
+	if (prefix !== undefined) validateSqliteSchemaPrefix(prefix);
+	const actualPrefix = prefix ?? "";
+	return {
+		tables: prefixIdentifiers(TABLE_NAMES, actualPrefix),
+		indexes: prefixIdentifiers(INDEX_NAMES, actualPrefix),
+	};
+}
+
+export const DEFAULT_SQLITE_SCHEMA = createSqliteSchema();

third_party/pi/packages/durable/src/storage/sqlite/storage.ts

@@ -0,0 +1,957 @@
+import type { Context, JsonValue } from "@earendil-works/chord";
+import { apply, type Op } from "@earendil-works/chord/delta";
+import { StorageRequestError } from "../../errors.ts";
+import { idFromNumber, seqFromNumber } from "../../ids.ts";
+import type {
+	ConversationId,
+	ConversationQuery,
+	ConversationRecord,
+	Cursor,
+	DocumentAddress,
+	DocumentContent,
+	DocumentCreate,
+	DocumentId,
+	DocumentPoint,
+	DocumentQuery,
+	DocumentRecord,
+	EntryId,
+	EntryQuery,
+	EntryRecord,
+	Id,
+	JsonObject,
+	Page,
+	ScanOrder,
+	Seq,
+	Storage,
+	StorageWrite,
+	StoredDocument,
+	SubmissionId,
+	SubmissionQuery,
+	SubmissionRecord,
+	TaskId,
+	TaskQuery,
+	TaskRecord,
+} from "../../types.ts";
+import { nextCursor, type ScanStart, scanStart } from "../scan.ts";
+import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "./database.ts";
+import { applySqliteMigrationsForSchema } from "./migrations.ts";
+import { createSqliteSchema, type SqliteSchema } from "./schema.ts";
+
+type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
+type TableName = "conversation" | "entry" | "task" | "submission" | "document";
+type RecordIdRow = { readonly record_type: TableName };
+type JsonRow = { readonly record: string };
+type EntryJsonRow = { readonly record: string; readonly commit_seq: number };
+type RevisionRow = {
+	readonly seq: number;
+	readonly kind: DocumentContent["kind"];
+	readonly version: number;
+	readonly content: string;
+};
+type IdRow = { readonly id: number };
+type MetadataRow = { readonly next_id: string; readonly next_seq: number };
+type DocumentAction = {
+	create?: DocumentCreate;
+	copy?: Extract<StorageWrite, { readonly type: "document.copy" }>["source"];
+	content?: DocumentContent;
+	retire: boolean;
+};
+type ScopeColumns = {
+	readonly scopeKind: DocumentRecord["scope"]["kind"];
+	readonly ownerId: number;
+};
+
+export type SqliteStorageOptions = {
+	/**
+	 * Prefix for every table and index in the durable schema. It must be a SQLite identifier and must not use
+	 * SQLite's reserved `sqlite_` prefix. Existing data is not migrated when this changes.
+	 */
+	readonly prefix?: string;
+};
+
+const parseJson = <T>(value: string): T => JSON.parse(value) as T;
+const encodeJson = (value: unknown): string => JSON.stringify(value) as string;
+// Some SQLite bindings replace lone UTF-16 surrogates. JSON encoding keeps indexed identities lossless.
+const encodeIndexedString = (value: string): string => JSON.stringify(value);
+
+const cursorId = <I extends Id<string>>(cursor: Cursor | undefined): I | undefined => {
+	const after = cursor?.after;
+	if (after === undefined) return undefined;
+	if (typeof after !== "number" || !Number.isSafeInteger(after))
+		throw new StorageRequestError("Invalid storage cursor");
+	return idFromNumber<I>(after);
+};
+
+const page = <T extends { readonly id: Id<string> }>(
+	values: readonly T[],
+	limit: number,
+	order: ScanOrder = "ascending",
+): Page<T, Cursor> => {
+	const items = values.slice(0, limit);
+	if (values.length <= limit) return { items };
+	return { items, next: nextCursor(items.at(-1)!.id, order) };
+};
+
+/** The ID condition, its parameter, and the `ORDER BY` direction of a table scan. */
+const scanSql = (start: ScanStart): { readonly clause: string; readonly param: number; readonly direction: string } =>
+	start.order === "ascending"
+		? { clause: "id > ?", param: start.after ?? -1, direction: "ASC" }
+		: { clause: "id < ?", param: start.after ?? Number.MAX_SAFE_INTEGER, direction: "DESC" };
+
+const scopeColumns = (scope: DocumentRecord["scope"]): ScopeColumns => {
+	switch (scope.kind) {
+		case "session":
+			return { scopeKind: "session", ownerId: 0 };
+		case "conversation":
+			return { scopeKind: "conversation", ownerId: scope.conversationId };
+		case "task":
+			return { scopeKind: "task", ownerId: scope.taskId };
+	}
+};
+
+const addressParts = (address: DocumentAddress | DocumentCreate | DocumentRecord) => {
+	const scope = scopeColumns(address.scope);
+	return {
+		kind: encodeIndexedString(address.kind),
+		...scope,
+		family: address.key === undefined ? 0 : 1,
+		keyValue: encodeIndexedString(address.key ?? ""),
+	};
+};
+
+const addressKey = (address: DocumentAddress | DocumentCreate | DocumentRecord): string => {
+	const parts = addressParts(address);
+	return JSON.stringify([parts.kind, parts.scopeKind, parts.ownerId, parts.family, parts.keyValue]);
+};
+
+const isAliveAt = (record: DocumentRecord, at: DocumentPoint): boolean => {
+	if (at === "current") return record.retiredAt === undefined;
+	return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt);
+};
+
+const isCurrentOnly = (record: DocumentRecord): boolean =>
+	record.scope.kind !== "conversation" || record.history === "latest";
+
+const writeId = (write: StorageWrite): Id<string> | undefined => {
+	switch (write.type) {
+		case "conversation":
+		case "entry":
+		case "task":
+		case "submission":
+			return write.value.id;
+		case "document.create":
+		case "document.copy":
+			return write.record.id;
+		case "document.change":
+		case "document.retire":
+			return undefined;
+	}
+};
+
+/** Portable SQLite implementation of the Pico storage contract. */
+export class SqliteStorage implements Storage {
+	private readonly db: SqliteDatabase;
+	private readonly schema: SqliteSchema;
+	private nextId: number;
+	private closed = false;
+	private closing: Promise<void> | undefined;
+	private admittedReads = 0;
+	private readsDrained: (() => void) | undefined;
+
+	private constructor(db: SqliteDatabase, schema: SqliteSchema, nextId: number) {
+		this.db = db;
+		this.schema = schema;
+		this.nextId = nextId;
+	}
+
+	/** Initialize storage over an owned SQLite database facade. */
+	static async open(db: SqliteDatabase, options: SqliteStorageOptions = {}): Promise<SqliteStorage> {
+		try {
+			const schema = createSqliteSchema(options.prefix);
+			await applySqliteMigrationsForSchema(db, schema);
+			const metadata = await db.get<MetadataRow>(
+				`SELECT next_id, next_seq FROM ${schema.tables.durableMetadata} WHERE singleton = 1`,
+			);
+			if (metadata === undefined) throw new Error("Durable SQLite metadata is missing");
+			return new SqliteStorage(db, schema, Number(metadata.next_id));
+		} catch (error) {
+			try {
+				await db.close();
+			} catch {
+				// Preserve the initialization failure.
+			}
+			throw error;
+		}
+	}
+
+	async commit(writes: readonly StorageWrite[], _context: Context): Promise<Seq> {
+		this.assertOpen();
+		const documentActions = this.prepareDocumentActions(writes);
+		const candidateNextId = this.candidateNextId(writes);
+		const seq = await this.db.transaction(async (transaction) => {
+			const metadata = await transaction.get<MetadataRow>(
+				`SELECT next_id, next_seq FROM ${this.schema.tables.durableMetadata} WHERE singleton = 1`,
+			);
+			if (metadata === undefined) throw new Error("Durable SQLite metadata is missing");
+			const committedSeq = seqFromNumber(metadata.next_seq);
+			await this.checkGlobalIds(transaction, writes);
+			await this.checkDocumentActions(transaction, documentActions);
+			for (const write of writes) await this.applyTableWrite(transaction, write, committedSeq);
+			await this.applyDocumentActions(transaction, documentActions, committedSeq);
+			await transaction.run(
+				`UPDATE ${this.schema.tables.durableMetadata} SET next_id = ?, next_seq = ? WHERE singleton = 1`,
+				String(Math.max(Number(metadata.next_id), candidateNextId)),
+				committedSeq + 1,
+			);
+			return committedSeq;
+		});
+		this.nextId = Math.max(this.nextId, candidateNextId);
+		return seq;
+	}
+
+	async mintId<I extends Id<string>>(): Promise<I> {
+		this.assertOpen();
+		if (!Number.isSafeInteger(this.nextId)) throw new Error("ID space is exhausted");
+		return idFromNumber<I>(this.nextId++);
+	}
+
+	async conversation(id: ConversationId, _context: Context): Promise<ConversationRecord | undefined> {
+		this.assertOpen();
+		const row = await this.db.get<JsonRow>(`SELECT record FROM ${this.schema.tables.conversations} WHERE id = ?`, id);
+		return row === undefined ? undefined : parseJson<ConversationRecord>(row.record);
+	}
+
+	async scanConversations(
+		query: ConversationQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<ConversationRecord, Cursor>> {
+		this.assertOpen();
+		const start = scanStart(query.order, cursor, "ascending");
+		const scan = scanSql(start);
+		const clauses = [scan.clause];
+		const params: SqliteValue[] = [scan.param];
+		if (query.ownerConversationId !== undefined) {
+			clauses.push("owner_conversation_id = ?");
+			params.push(query.ownerConversationId);
+		}
+		if (query.ownerTaskId !== undefined) {
+			clauses.push("owner_task_id = ?");
+			params.push(query.ownerTaskId);
+		}
+		params.push(limit + 1);
+		const rows = await this.db.all<JsonRow>(
+			`SELECT record FROM ${this.schema.tables.conversations} WHERE ${clauses.join(" AND ")} ORDER BY id ${scan.direction} LIMIT ?`,
+			...params,
+		);
+		return page(
+			rows.map((row) => parseJson<ConversationRecord>(row.record)),
+			limit,
+			start.order,
+		);
+	}
+
+	entry(id: EntryId, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
+	entry(
+		conversationId: ConversationId,
+		id: EntryId,
+		context: Context,
+	): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
+	entry(
+		idOrConversationId: EntryId | ConversationId,
+		idOrContext: EntryId | Context,
+		context?: Context,
+	): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined> {
+		return this.admitRead(() => this.readEntry(idOrConversationId, idOrContext, context));
+	}
+
+	findLatestHeadMarker(
+		conversationId: ConversationId,
+		atOrBeforeEntryId: EntryId | undefined,
+		_context: Context,
+	): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> {
+		return this.admitRead(() => this.readLatestHeadMarker(conversationId, atOrBeforeEntryId));
+	}
+
+	scanEntries(
+		query: EntryQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<EntryRecord, Cursor>> {
+		return this.admitRead(() => this.readEntries(query, limit, cursor));
+	}
+
+	private async readEntry(
+		idOrConversationId: EntryId | ConversationId,
+		idOrContext: EntryId | Context,
+		context?: Context,
+	): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined> {
+		const id =
+			context === undefined
+				? idFromNumber<EntryId>(idOrConversationId)
+				: typeof idOrContext === "number"
+					? idFromNumber<EntryId>(idOrContext)
+					: undefined;
+		if (id === undefined) throw new StorageRequestError("Storage.entry() requires an entry ID");
+		let conversation: ConversationRecord | undefined;
+		if (context !== undefined) {
+			const conversationId = idFromNumber<ConversationId>(idOrConversationId);
+			conversation = await this.readConversation(conversationId);
+			if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${conversationId}`);
+		}
+		const row = await this.db.get<EntryJsonRow>(
+			`SELECT record, commit_seq FROM ${this.schema.tables.entries} WHERE id = ?`,
+			id,
+		);
+		if (row === undefined) return undefined;
+		const entry = parseJson<EntryRecord>(row.record);
+		if (conversation !== undefined) {
+			let upperEntryId = Number.POSITIVE_INFINITY;
+			while (conversation.id !== entry.conversationId) {
+				if (conversation.parent === undefined) return undefined;
+				upperEntryId = Math.min(upperEntryId, conversation.parent.at);
+				conversation = (await this.readConversation(conversation.parent.conversationId))!;
+			}
+			if (entry.id > upperEntryId) return undefined;
+		}
+		return { entry, commitSeq: seqFromNumber(row.commit_seq) };
+	}
+
+	private async readLatestHeadMarker(
+		conversationId: ConversationId,
+		atOrBeforeEntryId: EntryId | undefined,
+	): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> {
+		let conversation = await this.readConversation(conversationId);
+		if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${conversationId}`);
+		let upper: number | undefined = atOrBeforeEntryId;
+		while (true) {
+			const row =
+				upper === undefined
+					? await this.db.get<JsonRow>(
+							`SELECT record FROM ${this.schema.tables.entries} WHERE conversation_id = ? AND head IS NOT NULL ORDER BY id DESC LIMIT 1`,
+							conversation.id,
+						)
+					: await this.db.get<JsonRow>(
+							`SELECT record FROM ${this.schema.tables.entries} WHERE conversation_id = ? AND head IS NOT NULL AND id <= ? ORDER BY id DESC LIMIT 1`,
+							conversation.id,
+							upper,
+						);
+			if (row !== undefined) return parseJson<EntryRecord & { readonly head: EntryId }>(row.record);
+			if (conversation.parent === undefined) return undefined;
+			upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at);
+			conversation = (await this.readConversation(conversation.parent.conversationId))!;
+		}
+	}
+
+	private async readEntries(
+		query: EntryQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+	): Promise<Page<EntryRecord, Cursor>> {
+		const { order, after } = scanStart(query.order, cursor, "descending");
+		if (order === "ascending") return this.readEntriesAscending(query, limit, after);
+		let conversation = await this.readConversation(query.conversationId);
+		if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${query.conversationId}`);
+		let upper: number | undefined = query.maxEntryId;
+		if (after !== undefined) upper = Math.min(upper ?? Number.MAX_SAFE_INTEGER, after - 1);
+		const values: EntryRecord[] = [];
+		while (true) {
+			const clauses = ["conversation_id = ?"];
+			const params: SqliteValue[] = [conversation.id];
+			if (query.minEntryId !== undefined) {
+				clauses.push("id >= ?");
+				params.push(query.minEntryId);
+			}
+			if (upper !== undefined) {
+				clauses.push("id <= ?");
+				params.push(upper);
+			}
+			params.push(limit + 1 - values.length);
+			const rows = await this.db.all<JsonRow>(
+				`SELECT record FROM ${this.schema.tables.entries} WHERE ${clauses.join(" AND ")} ORDER BY id DESC LIMIT ?`,
+				...params,
+			);
+			values.push(...rows.map((row) => parseJson<EntryRecord>(row.record)));
+			if (values.length > limit || conversation.parent === undefined) break;
+			upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at);
+			if (query.minEntryId !== undefined && upper < query.minEntryId) break;
+			conversation = (await this.readConversation(conversation.parent.conversationId))!;
+		}
+		return page(values, limit, "descending");
+	}
+
+	/** Oldest first: the fork chain's segments from the root conversation forward, each up to its fork point. */
+	private async readEntriesAscending(
+		query: EntryQuery,
+		limit: number,
+		after: number | undefined,
+	): Promise<Page<EntryRecord, Cursor>> {
+		const segments: { readonly conversationId: ConversationId; readonly upper: number | undefined }[] = [];
+		let conversation = await this.readConversation(query.conversationId);
+		if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${query.conversationId}`);
+		let upper: number | undefined = query.maxEntryId;
+		while (true) {
+			segments.push({ conversationId: conversation.id, upper });
+			if (conversation.parent === undefined) break;
+			upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at);
+			if (query.minEntryId !== undefined && upper < query.minEntryId) break;
+			conversation = (await this.readConversation(conversation.parent.conversationId))!;
+		}
+		let lower: number | undefined = query.minEntryId;
+		if (after !== undefined) lower = Math.max(lower ?? after + 1, after + 1);
+		const values: EntryRecord[] = [];
+		for (const segment of segments.reverse()) {
+			const clauses = ["conversation_id = ?"];
+			const params: SqliteValue[] = [segment.conversationId];
+			if (lower !== undefined) {
+				clauses.push("id >= ?");
+				params.push(lower);
+			}
+			if (segment.upper !== undefined) {
+				clauses.push("id <= ?");
+				params.push(segment.upper);
+			}
+			params.push(limit + 1 - values.length);
+			const rows = await this.db.all<JsonRow>(
+				`SELECT record FROM ${this.schema.tables.entries} WHERE ${clauses.join(" AND ")} ORDER BY id ASC LIMIT ?`,
+				...params,
+			);
+			values.push(...rows.map((row) => parseJson<EntryRecord>(row.record)));
+			if (values.length > limit) break;
+		}
+		return page(values, limit, "ascending");
+	}
+
+	async task(id: TaskId, _context: Context): Promise<StoredTask | undefined> {
+		this.assertOpen();
+		const row = await this.db.get<JsonRow>(`SELECT record FROM ${this.schema.tables.tasks} WHERE id = ?`, id);
+		return row === undefined ? undefined : parseJson<StoredTask>(row.record);
+	}
+
+	async scanTasks(
+		query: TaskQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<StoredTask, Cursor>> {
+		this.assertOpen();
+		const start = scanStart(query.order, cursor, "ascending");
+		const scan = scanSql(start);
+		const clauses = [scan.clause];
+		const params: SqliteValue[] = [scan.param];
+		if (query.conversationId !== undefined) {
+			clauses.push("conversation_id = ?");
+			params.push(query.conversationId);
+		}
+		if (query.kind !== undefined) {
+			clauses.push("kind = ?");
+			params.push(encodeIndexedString(query.kind));
+		}
+		if (query.status !== undefined) {
+			clauses.push("status = ?");
+			params.push(query.status);
+		}
+		if (query.abortRequested !== undefined) {
+			clauses.push("abort_requested = ?");
+			params.push(query.abortRequested ? 1 : 0);
+		}
+		if (query.background !== undefined) {
+			clauses.push("background = ?");
+			params.push(query.background ? 1 : 0);
+		}
+		params.push(limit + 1);
+		const rows = await this.db.all<JsonRow>(
+			`SELECT record FROM ${this.schema.tables.tasks} WHERE ${clauses.join(" AND ")} ORDER BY id ${scan.direction} LIMIT ?`,
+			...params,
+		);
+		return page(
+			rows.map((row) => parseJson<StoredTask>(row.record)),
+			limit,
+			start.order,
+		);
+	}
+
+	async submission(id: SubmissionId, _context: Context): Promise<SubmissionRecord | undefined> {
+		this.assertOpen();
+		const row = await this.db.get<JsonRow>(`SELECT record FROM ${this.schema.tables.submissions} WHERE id = ?`, id);
+		return row === undefined ? undefined : parseJson<SubmissionRecord>(row.record);
+	}
+
+	async scanSubmissions(
+		query: SubmissionQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<SubmissionRecord, Cursor>> {
+		this.assertOpen();
+		const start = scanStart(query.order, cursor, "ascending");
+		const scan = scanSql(start);
+		const clauses = [scan.clause];
+		const params: SqliteValue[] = [scan.param];
+		if (query.conversationId !== undefined) {
+			clauses.push("conversation_id = ?");
+			params.push(query.conversationId);
+		}
+		if (query.status !== undefined) {
+			clauses.push("status = ?");
+			params.push(query.status);
+		}
+		params.push(limit + 1);
+		const rows = await this.db.all<JsonRow>(
+			`SELECT record FROM ${this.schema.tables.submissions} WHERE ${clauses.join(" AND ")} ORDER BY id ${scan.direction} LIMIT ?`,
+			...params,
+		);
+		return page(
+			rows.map((row) => parseJson<SubmissionRecord>(row.record)),
+			limit,
+			start.order,
+		);
+	}
+
+	async submissionByRequest(
+		conversationId: ConversationId,
+		requestId: string,
+		_context: Context,
+	): Promise<SubmissionRecord | undefined> {
+		this.assertOpen();
+		const row = await this.db.get<JsonRow>(
+			`SELECT record FROM ${this.schema.tables.submissions} WHERE conversation_id = ? AND request_id = ?`,
+			conversationId,
+			encodeIndexedString(requestId),
+		);
+		return row === undefined ? undefined : parseJson<SubmissionRecord>(row.record);
+	}
+
+	async findDocument(
+		address: DocumentAddress,
+		at: DocumentPoint,
+		_context: Context,
+	): Promise<DocumentRecord | undefined> {
+		this.assertOpen();
+		const parts = addressParts(address);
+		const sql =
+			at === "current"
+				? `SELECT record FROM ${this.schema.tables.documents}
+					WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ?
+					AND retired_at IS NULL ORDER BY created_at DESC LIMIT 1`
+				: `SELECT record FROM ${this.schema.tables.documents}
+					WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ?
+					AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?)
+					ORDER BY created_at DESC LIMIT 1`;
+		const params: SqliteValue[] = [parts.kind, parts.scopeKind, parts.ownerId, parts.family, parts.keyValue];
+		if (at !== "current") params.push(at, at);
+		const row = await this.db.get<JsonRow>(sql, ...params);
+		return row === undefined ? undefined : parseJson<DocumentRecord>(row.record);
+	}
+
+	async document(id: DocumentId, at: DocumentPoint, _context: Context): Promise<StoredDocument | undefined> {
+		this.assertOpen();
+		// The record and revision queries must observe one committed state; a commit between them can replace the base.
+		return this.db.transaction((transaction) => this.materializeDocument(transaction, id, at));
+	}
+
+	async scanDocuments(
+		query: DocumentQuery,
+		limit: number,
+		cursor: Cursor | undefined,
+		_context: Context,
+	): Promise<Page<DocumentRecord, Cursor>> {
+		this.assertOpen();
+		const scope = scopeColumns(query.scope);
+		const clauses = ["scope_kind = ?", "owner_id = ?", "id > ?"];
+		const params: SqliteValue[] = [scope.scopeKind, scope.ownerId, cursorId(cursor) ?? -1];
+		if (query.kind !== undefined) {
+			clauses.push("kind = ?");
+			params.push(encodeIndexedString(query.kind));
+		}
+		if (query.at === "current") {
+			clauses.push("retired_at IS NULL");
+		} else {
+			clauses.push("created_at <= ?", "(retired_at IS NULL OR retired_at > ?)");
+			params.push(query.at, query.at);
+		}
+		params.push(limit + 1);
+		const rows = await this.db.all<JsonRow>(
+			`SELECT record FROM ${this.schema.tables.documents} WHERE ${clauses.join(" AND ")} ORDER BY id LIMIT ?`,
+			...params,
+		);
+		return page(
+			rows.map((row) => parseJson<DocumentRecord>(row.record)),
+			limit,
+		);
+	}
+
+	close(_context: Context): Promise<void> {
+		if (this.closing === undefined) {
+			this.closed = true;
+			this.closing = this.closeDatabase();
+		}
+		return this.closing;
+	}
+
+	private async closeDatabase(): Promise<void> {
+		if (this.admittedReads > 0) {
+			await new Promise<void>((resolve) => {
+				this.readsDrained = resolve;
+			});
+		}
+		await this.db.close();
+	}
+
+	/**
+	 * Run a read that issues several queries. Close waits for admitted reads, so their later queries never reach a
+	 * closed database. Single-query reads and transactions are already ordered before close by the database.
+	 */
+	private async admitRead<T>(read: () => Promise<T>): Promise<T> {
+		this.assertOpen();
+		this.admittedReads++;
+		try {
+			return await read();
+		} finally {
+			if (--this.admittedReads === 0) this.readsDrained?.();
+		}
+	}
+
+	private async readConversation(id: ConversationId): Promise<ConversationRecord | undefined> {
+		const row = await this.db.get<JsonRow>(`SELECT record FROM ${this.schema.tables.conversations} WHERE id = ?`, id);
+		return row === undefined ? undefined : parseJson<ConversationRecord>(row.record);
+	}
+
+	private async materializeDocument(
+		executor: SqliteExecutor,
+		id: DocumentId,
+		at: DocumentPoint,
+	): Promise<StoredDocument | undefined> {
+		const row = await executor.get<JsonRow>(`SELECT record FROM ${this.schema.tables.documents} WHERE id = ?`, id);
+		if (row === undefined) return undefined;
+		const record = parseJson<DocumentRecord>(row.record);
+		if (at !== "current" && isCurrentOnly(record)) {
+			throw new StorageRequestError(`Document ${id} does not retain historical content`);
+		}
+		if (!isAliveAt(record, at)) return undefined;
+		const upper = at === "current" ? Number.MAX_SAFE_INTEGER : at;
+		const base = await executor.get<RevisionRow>(
+			`SELECT seq, kind, version, content FROM ${this.schema.tables.documentRevisions}
+				WHERE document_id = ? AND kind = 'base' AND seq <= ? ORDER BY seq DESC LIMIT 1`,
+			id,
+			upper,
+		);
+		if (base === undefined) throw new Error(`Document ${id} is missing a required base`);
+		let value = parseJson<JsonObject>(base.content);
+		const tail = await executor.all<RevisionRow>(
+			`SELECT seq, kind, version, content FROM ${this.schema.tables.documentRevisions}
+				WHERE document_id = ? AND seq > ? AND seq <= ? ORDER BY seq`,
+			id,
+			base.seq,
+			upper,
+		);
+		for (const revision of tail) {
+			if (revision.kind !== "delta" || revision.version !== base.version) {
+				throw new Error(`Document ${id} crosses a stored version boundary without a base`);
+			}
+			value = apply(value, parseJson<readonly Op[]>(revision.content)) as JsonObject;
+		}
+		return { record, version: base.version, value, deltasSinceBase: tail.length };
+	}
+
+	private candidateNextId(writes: readonly StorageWrite[]): number {
+		let nextId = this.nextId;
+		for (const write of writes) {
+			const id = writeId(write);
+			if (id !== undefined) nextId = Math.max(nextId, id + 1);
+		}
+		return nextId;
+	}
+
+	private async checkGlobalIds(executor: SqliteExecutor, writes: readonly StorageWrite[]): Promise<void> {
+		const claimed = new Map<Id<string>, TableName>();
+		for (const write of writes) {
+			if (write.type === "document.change" || write.type === "document.retire") continue;
+			const document = write.type === "document.create" || write.type === "document.copy";
+			const table: TableName = document ? "document" : write.type;
+			const id = document ? write.record.id : write.value.id;
+			const existing = (
+				await executor.get<RecordIdRow>(`SELECT record_type FROM ${this.schema.tables.recordIds} WHERE id = ?`, id)
+			)?.record_type;
+			const earlier = claimed.get(id);
+			if (table === "conversation" || table === "entry" || table === "document") {
+				if (existing !== undefined) throw new Error(`ID ${id} already belongs to ${existing}`);
+				if (earlier !== undefined) throw new Error(`ID ${id} is written more than once`);
+			} else {
+				if (existing !== undefined && existing !== table)
+					throw new Error(`ID ${id} already belongs to ${existing}`);
+				if (earlier !== undefined && earlier !== table) throw new Error(`ID ${id} is written as two record types`);
+			}
+			claimed.set(id, table);
+		}
+	}
+
+	private prepareDocumentActions(writes: readonly StorageWrite[]): Map<DocumentId, DocumentAction> {
+		const actions = new Map<DocumentId, DocumentAction>();
+		for (const write of writes) {
+			if (
+				write.type !== "document.create" &&
+				write.type !== "document.copy" &&
+				write.type !== "document.change" &&
+				write.type !== "document.retire"
+			) {
+				continue;
+			}
+			const id = write.type === "document.create" || write.type === "document.copy" ? write.record.id : write.id;
+			let action = actions.get(id);
+			if (action === undefined) {
+				action = { retire: false };
+				actions.set(id, action);
+			}
+			switch (write.type) {
+				case "document.create":
+					if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) {
+						throw new Error(`Document ${id} has more than one content command`);
+					}
+					action.create = write.record;
+					action.content = write.content;
+					break;
+				case "document.copy":
+					if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) {
+						throw new Error(`Document ${id} has more than one content command`);
+					}
+					action.create = write.record;
+					action.copy = write.source;
+					break;
+				case "document.change":
+					if (action.content !== undefined || action.copy !== undefined) {
+						throw new Error(`Document ${id} has more than one content command`);
+					}
+					action.content = write.content;
+					break;
+				case "document.retire":
+					if (action.retire) throw new Error(`Document ${id} is retired more than once`);
+					action.retire = true;
+					break;
+			}
+		}
+		return actions;
+	}
+
+	private async checkDocumentActions(
+		executor: SqliteExecutor,
+		actions: ReadonlyMap<DocumentId, DocumentAction>,
+	): Promise<void> {
+		const liveCounts = new Map<string, number>();
+		for (const [id, action] of actions) {
+			if (action.copy !== undefined && actions.has(action.copy.id)) {
+				throw new Error(`Document copy ${id} source is changed in the copy batch`);
+			}
+			const row = await executor.get<JsonRow>(`SELECT record FROM ${this.schema.tables.documents} WHERE id = ?`, id);
+			const existing = row === undefined ? undefined : parseJson<DocumentRecord>(row.record);
+			if (action.create === undefined && existing === undefined) throw new Error(`Unknown document: ${id}`);
+			if (action.create !== undefined && existing !== undefined) throw new Error(`Document ${id} already exists`);
+			if (existing?.retiredAt !== undefined) throw new Error(`Document ${id} is retired`);
+			if (action.content?.kind === "delta") {
+				const previous = await executor.get<{ readonly version: number }>(
+					`SELECT version FROM ${this.schema.tables.documentRevisions} WHERE document_id = ? ORDER BY seq DESC LIMIT 1`,
+					id,
+				);
+				if (previous === undefined) throw new Error(`Document ${id} delta has no base`);
+				if (previous.version !== action.content.version) {
+					throw new Error(`Document ${id} version transition requires a base`);
+				}
+			}
+			const record = action.create ?? existing!;
+			const key = addressKey(record);
+			let live = liveCounts.get(key);
+			if (live === undefined) live = (await this.currentDocumentId(executor, record)) === undefined ? 0 : 1;
+			if (action.retire && existing !== undefined) live--;
+			if (action.create !== undefined && !action.retire) live++;
+			liveCounts.set(key, live);
+		}
+		for (const live of liveCounts.values()) {
+			if (live > 1) throw new Error("Document address already has a current incarnation");
+		}
+	}
+
+	private async currentDocumentId(
+		executor: SqliteExecutor,
+		address: DocumentAddress | DocumentCreate | DocumentRecord,
+	): Promise<DocumentId | undefined> {
+		const parts = addressParts(address);
+		const id = (
+			await executor.get<IdRow>(
+				`SELECT id FROM ${this.schema.tables.documents}
+				WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ? AND retired_at IS NULL
+				LIMIT 1`,
+				parts.kind,
+				parts.scopeKind,
+				parts.ownerId,
+				parts.family,
+				parts.keyValue,
+			)
+		)?.id;
+		return id === undefined ? undefined : idFromNumber<DocumentId>(id);
+	}
+
+	private async applyTableWrite(executor: SqliteExecutor, write: StorageWrite, seq: Seq): Promise<void> {
+		switch (write.type) {
+			case "conversation":
+				await this.claimId(executor, write.value.id, "conversation");
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.conversations} (id, owner_conversation_id, owner_task_id, record) VALUES (?, ?, ?, ?)`,
+					write.value.id,
+					write.value.owner?.conversationId ?? null,
+					write.value.owner?.taskId ?? null,
+					encodeJson(write.value),
+				);
+				break;
+			case "entry":
+				await this.claimId(executor, write.value.id, "entry");
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.entries} (id, conversation_id, head, commit_seq, record) VALUES (?, ?, ?, ?, ?)`,
+					write.value.id,
+					write.value.conversationId,
+					write.value.head ?? null,
+					seq,
+					encodeJson(write.value),
+				);
+				break;
+			case "task":
+				await this.claimId(executor, write.value.id, "task");
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.tasks} (id, conversation_id, kind, status, abort_requested, background, record)
+						VALUES (?, ?, ?, ?, ?, ?, ?)
+						ON CONFLICT(id) DO UPDATE SET conversation_id = excluded.conversation_id, kind = excluded.kind,
+						status = excluded.status, abort_requested = excluded.abort_requested,
+						background = excluded.background, record = excluded.record`,
+					write.value.id,
+					write.value.conversationId,
+					encodeIndexedString(write.value.kind),
+					write.value.state.status,
+					write.value.abortRequested ? 1 : 0,
+					write.value.background ? 1 : 0,
+					encodeJson(write.value),
+				);
+				break;
+			case "submission":
+				await this.claimId(executor, write.value.id, "submission");
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.submissions} (id, conversation_id, request_id, status, record) VALUES (?, ?, ?, ?, ?)
+						ON CONFLICT(id) DO UPDATE SET conversation_id = excluded.conversation_id,
+						request_id = excluded.request_id, status = excluded.status, record = excluded.record`,
+					write.value.id,
+					write.value.conversationId,
+					write.value.requestId === undefined ? null : encodeIndexedString(write.value.requestId),
+					write.value.status,
+					encodeJson(write.value),
+				);
+				break;
+			case "document.create":
+			case "document.copy":
+			case "document.change":
+			case "document.retire":
+				break;
+		}
+	}
+
+	private async claimId(executor: SqliteExecutor, id: Id<string>, table: TableName): Promise<void> {
+		await executor.run(
+			`INSERT OR IGNORE INTO ${this.schema.tables.recordIds} (id, record_type) VALUES (?, ?)`,
+			id,
+			table,
+		);
+	}
+
+	private async applyDocumentActions(
+		executor: SqliteExecutor,
+		actions: ReadonlyMap<DocumentId, DocumentAction>,
+		seq: Seq,
+	): Promise<void> {
+		for (const [id, action] of actions) {
+			let content = action.content;
+			if (action.copy !== undefined) {
+				try {
+					const stored = await this.materializeDocument(executor, action.copy.id, action.copy.at);
+					if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`);
+					const create = action.create!;
+					if (
+						stored.record.scope.kind !== "conversation" ||
+						create.scope.kind !== "conversation" ||
+						stored.record.kind !== create.kind ||
+						stored.record.key !== create.key ||
+						stored.record.history !== create.history ||
+						stored.record.fork !== create.fork
+					) {
+						throw new Error(`Fork source document ${action.copy.id} does not match the copied record`);
+					}
+					content = { kind: "base", version: stored.version, value: stored.value };
+				} catch (error) {
+					throw new Error(`Document copy ${id} was rejected`, { cause: error });
+				}
+			}
+			let record: DocumentRecord;
+			if (action.create !== undefined) {
+				record = {
+					...action.create,
+					createdAt: seq,
+					...(action.retire ? { retiredAt: seq } : {}),
+				};
+				const parts = addressParts(record);
+				await this.claimId(executor, id, "document");
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.documents}
+						(id, kind, family, key_value, scope_kind, owner_id, created_at, retired_at, record)
+						VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+					id,
+					parts.kind,
+					parts.family,
+					parts.keyValue,
+					parts.scopeKind,
+					parts.ownerId,
+					seq,
+					action.retire ? seq : null,
+					encodeJson(record),
+				);
+			} else {
+				const row = (await executor.get<JsonRow>(
+					`SELECT record FROM ${this.schema.tables.documents} WHERE id = ?`,
+					id,
+				))!;
+				record = parseJson<DocumentRecord>(row.record);
+			}
+
+			if (content !== undefined) {
+				if (content.kind === "base" && isCurrentOnly(record)) {
+					await executor.run(`DELETE FROM ${this.schema.tables.documentRevisions} WHERE document_id = ?`, id);
+				}
+				const encodedContent = content.kind === "base" ? encodeJson(content.value) : encodeJson(content.ops);
+				await executor.run(
+					`INSERT INTO ${this.schema.tables.documentRevisions} (document_id, seq, kind, version, content) VALUES (?, ?, ?, ?, ?)`,
+					id,
+					seq,
+					content.kind,
+					content.version,
+					encodedContent,
+				);
+			}
+
+			if (action.retire) {
+				if (action.create === undefined) {
+					record = { ...record, retiredAt: seq };
+					await executor.run(
+						`UPDATE ${this.schema.tables.documents} SET retired_at = ?, record = ? WHERE id = ?`,
+						seq,
+						encodeJson(record),
+						id,
+					);
+				}
+				if (isCurrentOnly(record)) {
+					await executor.run(`DELETE FROM ${this.schema.tables.documentRevisions} WHERE document_id = ?`, id);
+				}
+			}
+		}
+	}
+
+	private assertOpen(): void {
+		if (this.closed) throw new Error("SqliteStorage is closed");
+	}
+}