alexion/dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(nixos): build vendored Pi #46

Merged
alexion merged 3 commits from import-pi-0.85.1 into main 2026-09-14 10:51:32 -04:00
Conversation 0 Commits 3 Files Changed 1675 +397675 -458
1675 changed files with 397675 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
.pi/extensions/herdr-agent-state.ts
View File

@@ -1,287 +0,0 @@
// installed by herdr
// managed by herdr; reinstalling or updating the integration overwrites this file.
// add custom hooks/plugins beside this file instead of editing it.
// HERDR_INTEGRATION_ID=pi
// HERDR_INTEGRATION_VERSION=6
// @ts-nocheck
import net from "node:net";
const HERDR_ENV = process.env.HERDR_ENV;
const socketPath = process.env.HERDR_SOCKET_PATH;
const socketEndpoint =
process.platform === "win32" && socketPath ? `\\\\.\\pipe\\${socketPath}` : socketPath;
const paneId = process.env.HERDR_PANE_ID;
const source = "herdr:pi";
function enabled() {
return HERDR_ENV === "1" && !!socketPath && !!paneId;
}
function sendRequestAttempt(request: unknown, timeoutMs: number): Promise<boolean> {
if (!enabled()) {
return Promise.resolve(true);
}
return new Promise((resolve) => {
let done = false;
let timeout: ReturnType<typeof setTimeout> | undefined;
const finish = (delivered: boolean) => {
if (done) return;
done = true;
if (timeout) {
clearTimeout(timeout);
}
socket.destroy();
resolve(delivered);
};
const socket = net.createConnection(socketEndpoint!);
socket.on("error", () => finish(false));
socket.on("connect", () => socket.write(`${JSON.stringify(request)}\n`));
socket.on("data", () => finish(true));
socket.on("end", () => finish(false));
timeout = setTimeout(() => finish(false), timeoutMs);
timeout.unref?.();
});
}
async function sendRequest(request: unknown): Promise<void> {
if (await sendRequestAttempt(request, 500)) {
return;
}
await sendRequestAttempt(request, 1500);
}
type AgentState = "working" | "blocked" | "idle";
type QueuedState = {
state: AgentState;
message?: string;
seq: number;
};
let reportSeq = Date.now() * 1000;
let currentAgentSessionId: string | undefined;
let currentAgentSessionPath: string | undefined;
function nextReportSeq(): number {
reportSeq += 1;
return reportSeq;
}
function updateSessionRef(ctx: any): void {
try {
const file = ctx?.sessionManager?.getSessionFile?.();
currentAgentSessionPath =
typeof file === "string" && file.startsWith("/") ? file : undefined;
} catch {
currentAgentSessionPath = undefined;
}
try {
const id = ctx?.sessionManager?.getSessionId?.();
currentAgentSessionId = typeof id === "string" && id.length > 0 ? id : undefined;
} catch {
currentAgentSessionId = undefined;
}
}
function withSessionRef(params: Record<string, unknown>): Record<string, unknown> {
if (currentAgentSessionPath) {
return { ...params, agent_session_path: currentAgentSessionPath };
}
if (currentAgentSessionId) {
return { ...params, agent_session_id: currentAgentSessionId };
}
return params;
}
function currentSessionRef(): Record<string, unknown> | undefined {
if (currentAgentSessionPath) {
return { agent_session_path: currentAgentSessionPath };
}
if (currentAgentSessionId) {
return { agent_session_id: currentAgentSessionId };
}
return undefined;
}
function reportSession(sessionStartSource?: string): Promise<void> {
const sessionRef = currentSessionRef();
if (!sessionRef) {
return Promise.resolve();
}
return sendRequest({
id: `${source}:session:${Date.now()}:${Math.random().toString(36).slice(2)}`,
method: "pane.report_agent_session",
params: {
pane_id: paneId,
source,
agent: "pi",
seq: nextReportSeq(),
session_start_source: sessionStartSource,
...sessionRef,
},
});
}
function sendState(state: AgentState, message?: string, seq = nextReportSeq()): Promise<void> {
return sendRequest({
id: `${source}:${Date.now()}:${Math.random().toString(36).slice(2)}`,
method: "pane.report_agent",
params: withSessionRef({
pane_id: paneId,
source,
agent: "pi",
state,
message,
seq,
}),
});
}
function releaseAgent(): Promise<void> {
return sendRequest({
id: `${source}:release:${Date.now()}:${Math.random().toString(36).slice(2)}`,
method: "pane.release_agent",
params: {
pane_id: paneId,
source,
agent: "pi",
seq: nextReportSeq(),
},
});
}
function shouldReleaseOnSessionShutdown(event: any): boolean {
// Pi tears down and rebinds extension runtimes for internal lifecycle actions
// such as /reload, /new, /resume, and /fork. Those do not mean the pane's
// agent process has exited, and releasing hook authority there can suppress
// legitimate reports from the replacement runtime. Only a user/process quit
// should release Herdr's full-lifecycle authority.
const reason = event?.reason;
return reason === "quit";
}
let sendInFlight = false;
let queuedState: QueuedState | undefined;
function queueState(state: AgentState, message?: string): void {
queuedState = { state, message, seq: nextReportSeq() };
if (!sendInFlight) {
void drainStateQueue();
}
}
async function drainStateQueue(): Promise<void> {
if (sendInFlight) {
return;
}
sendInFlight = true;
try {
while (queuedState) {
const next = queuedState;
queuedState = undefined;
await sendState(next.state, next.message, next.seq);
}
} finally {
sendInFlight = false;
if (queuedState) {
void drainStateQueue();
}
}
}
export default function (pi) {
if (!enabled()) {
return;
}
let agentActive = false;
let blockedCount = 0;
let blockedMessage: string | undefined;
let lastState: AgentState | undefined;
let lastMessage: string | undefined;
let rootSession = false;
function desiredState() {
if (blockedCount > 0) {
return { state: "blocked" as const, message: blockedMessage };
}
if (agentActive) {
return { state: "working" as const, message: undefined };
}
return { state: "idle" as const, message: undefined };
}
function publishState(force = false) {
const next = desiredState();
if (!force && next.state === lastState && next.message === lastMessage) {
return;
}
lastState = next.state;
lastMessage = next.message;
queueState(next.state, next.message);
}
pi.events.on("herdr:blocked", (data) => {
if (!rootSession) {
return;
}
if (!data?.active) {
blockedCount = Math.max(0, blockedCount - 1);
if (blockedCount === 0) {
blockedMessage = undefined;
}
publishState();
return;
}
blockedCount += 1;
blockedMessage = data.label;
publishState();
});
pi.on("session_start", async (event, ctx) => {
if (ctx?.hasUI !== true) {
return;
}
rootSession = true;
updateSessionRef(ctx);
await reportSession(event?.reason);
// A reload can replace this extension mid-run without emitting another agent_start.
agentActive = ctx?.isIdle?.() === false;
publishState(true);
});
pi.on("agent_start", (_event, ctx) => {
if (!rootSession) {
return;
}
updateSessionRef(ctx);
void reportSession();
agentActive = true;
publishState();
});
pi.on("agent_settled", (_event, ctx) => {
if (!rootSession || ctx?.isIdle?.() !== true) {
return;
}
agentActive = false;
publishState();
});
pi.on("session_shutdown", async (event) => {
if (!rootSession) {
return;
}
if (shouldReleaseOnSessionShutdown(event)) {
await releaseAgent();
}
});
}

4
AGENTS.md
View File

@@ -110,3 +110,7 @@
absent from the pinned revision or from a different branch such as `nixpkgs-unstable`.
- OMP does not discover project skills from `.pi/skills`.
Put skills shared by Pi and OMP under `.agents/skills`, or configure OMP's `skills.customDirectories`.
- Pi is vendored under `third_party/pi` as a linear vendor import.
Do not use `git subtree add` or `git subtree pull` for Pi PR branches.
Gitea rebases the synthetic upstream root commit without the `third_party/pi` prefix.
Preserve `git-subtree-dir` and `git-subtree-split` trailers on Pi vendor commits.

45
README.md
View File

@@ -13,7 +13,8 @@ dev/ Monorepo root
├── src/ Projects
│ └── dotfiles/ Home and system configuration
└── third_party/ External source and vendoring support
└── crates/ Generated Rust crate sources and Bazel targets
├── crates/ Generated Rust crate sources and Bazel targets
└── pi/ Vendored Pi source
```
## Rust dependencies
@@ -44,3 +45,45 @@ deps = ["//third_party/crates:crate_name"]
Commit the generated files under `third_party/crates/`.
Do not edit generated files manually.
## Vendored Pi
Pi source lives in `third_party/pi/`.
The directory is a linear vendor import from upstream Pi.
Local fork changes are carried directly in that vendored tree.
Use the upstream tag that matches the target CLI release.
The CLI version lives in `packages/coding-agent/package.json`.
Do not use `git subtree add` or `git subtree pull` for Pi PR branches.
Those commands create a synthetic upstream root commit.
Gitea rebase applies that root commit without the `third_party/pi` prefix.
That causes add/add conflicts on repository root files.
Update the vendor tree with a linear commit:
```sh
git fetch https://github.com/earendil-works/pi.git refs/tags/v<version>
split="$(git rev-parse FETCH_HEAD)"
rm -rf third_party/pi
mkdir -p third_party/pi
git archive --format=tar "$split" | tar -C third_party/pi -xf -
git add third_party/pi
git commit \
-m "chore(vendor): import pi <version>" \
-m "Imported as a linear vendor commit so forge rebases do not replay the upstream root tree." \
-m "git-subtree-dir: third_party/pi" \
-m "git-subtree-split: $split"
```
Keep the vendor import or update commit separate from local edits.
Do not remove the `git-subtree-dir` and `git-subtree-split` trailers.
They record the upstream source commit for future updates.
After the upstream update, resolve conflicts in `third_party/pi/`.
Keep local fork changes as ordinary commits after the vendor update.
Then verify the package:
```sh
nix build .#nixosConfigurations.zeus.config.home-manager.users.alexion.programs.pi-coding-agent.package --no-link --print-out-paths
```

12
configs/nixos/modules/shell/pi/default.nix
View File

@@ -11,12 +11,10 @@ let
cfg = config.modules.shell.pi;
user = config.user.name;
piDir = "${config.users.users.${user}.home}/.pi/agent";
patchedPi = pkgs.pi-coding-agent.overrideAttrs (old: {
patches = (old.patches or [ ]) ++ [
./patches/pi-project-resource-ancestors.patch
./patches/pi-tool-lookup-validation.patch
];
});
vendoredPi = pkgs.pi-coding-agent.overrideAttrs {
src = ../../../../../third_party/pi;
patches = [ ];
};
in
{
options.modules.shell.pi = {
@@ -28,7 +26,7 @@ in
home-manager.users.${user} = {
programs.pi-coding-agent = {
enable = true;
package = patchedPi;
package = vendoredPi;
settings = {
defaultProvider = "openai-codex";

140
configs/nixos/modules/shell/pi/patches/pi-project-resource-ancestors.patch
View File

@@ -1,140 +0,0 @@
diff --git a/packages/coding-agent/src/core/package-manager.ts b/packages/coding-agent/src/core/package-manager.ts
--- a/packages/coding-agent/src/core/package-manager.ts
+++ b/packages/coding-agent/src/core/package-manager.ts
@@ -459,14 +459,14 @@ function findGitRepoRoot(startDir: string): string | null {
}
}
-function collectAncestorAgentsSkillDirs(startDir: string): string[] {
- const skillDirs: string[] = [];
+function collectAncestorConfigDirs(startDir: string, configDirName: string): string[] {
+ const configDirs: string[] = [];
const resolvedStartDir = resolve(startDir);
const gitRepoRoot = findGitRepoRoot(resolvedStartDir);
let dir = resolvedStartDir;
while (true) {
- skillDirs.push(join(dir, ".agents", "skills"));
+ configDirs.push(join(dir, configDirName));
if (gitRepoRoot && dir === gitRepoRoot) {
break;
}
@@ -477,7 +477,7 @@ function collectAncestorAgentsSkillDirs(startDir: string): string[] {
dir = parent;
}
- return skillDirs;
+ return configDirs;
}
function collectAutoPromptEntries(dir: string): string[] {
@@ -2388,16 +2388,13 @@ export class DefaultPackageManager implements PackageManager {
prompts: join(globalBaseDir, "prompts"),
themes: join(globalBaseDir, "themes"),
};
- const projectDirs = {
- extensions: join(projectBaseDir, "extensions"),
- skills: join(projectBaseDir, "skills"),
- prompts: join(projectBaseDir, "prompts"),
- themes: join(projectBaseDir, "themes"),
- };
const userAgentsSkillsDir = join(getHomeDir(), ".agents", "skills");
const projectTrusted = this.settingsManager.isProjectTrusted();
- const projectAgentsSkillDirs = projectTrusted
- ? collectAncestorAgentsSkillDirs(this.cwd).filter((dir) => resolve(dir) !== resolve(userAgentsSkillsDir))
+ const projectConfigDirs = projectTrusted
+ ? collectAncestorConfigDirs(this.cwd, CONFIG_DIR_NAME).filter((dir) => resolve(dir) !== resolve(globalBaseDir))
+ : [];
+ const projectAgentsDirs = projectTrusted
+ ? collectAncestorConfigDirs(this.cwd, ".agents").filter((dir) => resolve(dir) !== resolve(dirname(userAgentsSkillsDir)))
: [];
const addResources = (
@@ -2414,56 +2411,54 @@ export class DefaultPackageManager implements PackageManager {
}
};
- if (projectTrusted) {
- // Project extensions from .pi/
+ // Project resources from .pi/ in cwd and ancestors.
+ for (const configDir of projectConfigDirs) {
+ const configMetadata: PathMetadata = {
+ ...projectMetadata,
+ baseDir: configDir,
+ };
addResources(
"extensions",
- collectAutoExtensionEntries(projectDirs.extensions),
- projectMetadata,
+ collectAutoExtensionEntries(join(configDir, "extensions")),
+ configMetadata,
projectOverrides.extensions,
- projectBaseDir,
+ configDir,
);
-
- // Project skills from .pi/
addResources(
"skills",
- collectAutoSkillEntries(projectDirs.skills, "pi"),
- projectMetadata,
+ collectAutoSkillEntries(join(configDir, "skills"), "pi"),
+ configMetadata,
projectOverrides.skills,
- projectBaseDir,
+ configDir,
+ );
+ addResources(
+ "prompts",
+ collectAutoPromptEntries(join(configDir, "prompts")),
+ configMetadata,
+ projectOverrides.prompts,
+ configDir,
+ );
+ addResources(
+ "themes",
+ collectAutoThemeEntries(join(configDir, "themes")),
+ configMetadata,
+ projectOverrides.themes,
+ configDir,
);
}
- // Project skills from .agents/ (each with its own baseDir)
- for (const agentsSkillsDir of projectAgentsSkillDirs) {
- const agentsBaseDir = dirname(agentsSkillsDir); // the .agents directory
+ // Project skills from .agents/ in cwd and ancestors.
+ for (const agentsDir of projectAgentsDirs) {
const agentsMetadata: PathMetadata = {
...projectMetadata,
- baseDir: agentsBaseDir,
+ baseDir: agentsDir,
};
addResources(
"skills",
- collectAutoSkillEntries(agentsSkillsDir, "agents"),
+ collectAutoSkillEntries(join(agentsDir, "skills"), "agents"),
agentsMetadata,
projectOverrides.skills,
- agentsBaseDir,
- );
- }
-
- if (projectTrusted) {
- addResources(
- "prompts",
- collectAutoPromptEntries(projectDirs.prompts),
- projectMetadata,
- projectOverrides.prompts,
- projectBaseDir,
- );
- addResources(
- "themes",
- collectAutoThemeEntries(projectDirs.themes),
- projectMetadata,
- projectOverrides.themes,
- projectBaseDir,
+ agentsDir,
);
}

23
configs/nixos/modules/shell/pi/patches/pi-tool-lookup-validation.patch
View File

@@ -1,23 +0,0 @@
diff --git a/packages/coding-agent/src/utils/tools-manager.ts b/packages/coding-agent/src/utils/tools-manager.ts
--- a/packages/coding-agent/src/utils/tools-manager.ts 2026-08-01 18:41:36.970496010 -0400
+++ b/packages/coding-agent/src/utils/tools-manager.ts 2026-08-01 18:41:37.028186009 -0400
@@ -74,8 +74,8 @@
function commandExists(cmd: string): boolean {
try {
const result = spawnSync(cmd, ["--version"], { stdio: "pipe" });
- // Check for ENOENT error (command not found)
- return result.error === undefined || result.error === null;
+ // Downloaded generic Linux binaries can exist but fail on NixOS before they print a version.
+ return (result.error === undefined || result.error === null) && result.status === 0;
} catch {
return false;
}
@@ -88,7 +87,7 @@
// Check our tools directory first
const localPath = join(TOOLS_DIR, config.binaryName + (platform() === "win32" ? ".exe" : ""));
- if (existsSync(localPath)) {
+ if (existsSync(localPath) && commandExists(localPath)) {
return localPath;
}

23
third_party/pi/.gitattributes vendored Normal file
View File

@@ -0,0 +1,23 @@
# Default to LF for text files across the repo
* text=auto eol=lf
# Windows scripts should keep CRLF
*.bat text eol=crlf
*.cmd text eol=crlf
*.ps1 text eol=crlf
# Shell scripts should keep LF
*.sh text eol=lf
# Common binary assets
*.png binary
*.jpg binary
*.jpeg binary
*.gif binary
*.webp binary
*.ico binary
*.pdf binary
*.zip binary
*.gz binary
*.woff binary
*.woff2 binary

391
third_party/pi/.github/APPROVED_CONTRIBUTORS vendored Normal file
View File

@@ -0,0 +1,391 @@
# GitHub handles approved to bypass contribution auto-close
# Format: <username> <capability>
# capability:
# issue future issues stay open
# pr future issues and PRs stay open
herrnel pr
julien-c pr
barapa pr
alasano pr
aadishv pr
airtonix pr
aliou pr
aos pr
austinm911 pr
banteg pr
ben-vargas pr
butelo pr
can1357 pr
CarlosGtrz pr
cau1k pr
cmf pr
crcatala pr
Cursivez pr
cv pr
dannote pr
default-anton pr
dnouri pr
DronNick pr
enisdenjo pr
ferologics pr
fightbulc pr
ghoulr pr
gnattu pr
HACKE-RC pr
hewliyang pr
hjanuschka pr
iamd3vil pr
jblwilliams pr
joshp123 pr
jsinge97 pr
justram pr
kaofelix pr
kiliman pr
kim0 pr
lockmeister pr
LukeFost pr
lukele pr
m-box-mr pr
marckrenn pr
markusylisiurunen pr
mcinteerj pr
melihmucuk pr
mitsuhiko pr
mrexodia pr
nathyong pr
nickseelert pr
nicobailon pr
ninlds pr
ogulcancelik pr
patrick-kidger pr
paulbettner pr
Perlence pr
pjtf93 pr
prateekmedia pr
prathamdby pr
ribelo pr
richardgill pr
robinwander pr
ronyrus pr
roshanasingh4 pr
scutifer pr
skuridin pr
steipete pr
svkozak pr
tallshort pr
theBucky pr
thomasmhr pr
tiagoefreitas pr
timolins pr
tmustier pr
tudoroancea pr
unexge pr
vaayne pr
VaclavSynacek pr
vsabavat pr
w-winter pr
Whamp pr
WismutHansen pr
XesGaDeus pr
yevhen pr
badlogictest pr
terrorobe pr
zedrdave pr
mrud pr
toorusr pr
andresaraujo pr
lightningRalf pr
williballenthin pr
masonc15 pr
4h9fbZ pr
haoqixu pr
Graffioh pr
charles-cooper pr
emanuelst pr
juanibiapina pr
liby pr
pasky pr
odysseus0 pr
giuseppeg pr
michaelpersonal pr
academo pr
PriNova pr
semtexzv pr
jasonish pr
markusn pr
SamFold pr
Soleone pr
virtuald pr
NateSmyth pr
7Sageer pr
MatthieuBizien pr
sumeet pr
marchellodev pr
vedang pr
lucemia pr
mcollina pr
lajarre pr
smithbm2316 pr
drewburr pr
gordonhwc pr
deybhayden pr
tintinweb pr
asoules pr
zhahaoyu pr
in0vik pr
jtac pr
yzhg1983 pr
smcllns pr
dmmulroy pr
zmberber pr
andresvi94 pr
sudosubin pr
Mic92 pr
pmateusz pr
wirjo pr
jay-aye-see-kay pr
lucasmeijer pr
Evizero pr
ofa1 pr
crisog issue
mpazik pr
vekexasia pr
Michaelliv pr
cmraible pr
dljsjr pr
drio pr
jlaneve pr
tantara pr
Nutlope pr
xl0 pr
mdsjip pr
Exrun94 pr
marcbloech pr
pidalf pr
injaneity pr
thirtythreeforty pr
justinpbarnett pr
cristinaponcela pr
LooSik pr
mchenco pr
Phoen1xCode pr
louis030195 pr
technocidal pr
pandada8 pr
npupko issue
chrisvariety pr
maximilianzuern pr
brianmichel pr
abhinavmathur-atlan pr
mattiacerutti pr
josephyoung pr
mbazso pr
AJM10565 pr
DanielThomas pr
MichaelYochpaz pr
stephanmck pr
rolfvreijdenberger pr
psoukie pr
vastxie pr
ItsumoSeito pr
davidlifschitz pr
vdxz pr
dangooddd pr
Mearman pr
dodiego pr
any-victor pr
geraschenko pr
skhoroshavin pr
cyzlmh pr
xz-dev pr
rajp152k pr
affanali2k3 pr
ArcadiaLin pr
anilgulecha pr
DeviosLang pr
HarrodRen pr
aaronkyriesenbach pr
farid-fari pr
petrroll pr
vibeinging pr
DivineDominion pr
ananthakumaran pr
andrebreijao pr
anh-chu pr
rsaryev pr
QuintinShaw pr
R-Taneja pr
zaycruz pr
mteam88 pr
christianbasch pr
cpacker pr
rgarcia pr
renaudhartert-db pr
HyeokjaeLee pr
arajkumar pr
brianstanley pr
scruffymongrel pr
SI-RUI-ZHANG pr
sunnyyoung pr
muyiyr pr
hi-neason pr
acmerfight pr
tizmagik pr
jingtao-wisdomgraph pr
XRX193 pr
autopeasant pr
Snail-Turbo pr
futile pr
arasovic pr
xXJSONDeruloXx pr
Marvae pr
skkdevcraft pr
PierrunoYT pr
zhichli pr
wesleyzhangwq pr
dgokeeffe pr
vipentti pr
midastruth pr
Maximo-Guk pr
bigoldcat123 pr
johnatbasicas pr
powerfooI pr
yearth pr
pablasso pr
bilby91 pr
giannisCKS pr
Panoplos pr
haoyongchun1125-maker pr
gwokhou pr
gaoyk19 pr
cad0p pr
Jaaneek pr
CaiJichang212 pr
Mallikarjun-0 pr
wutongyuonce pr
Terminator666666 pr
y-nk pr
edverma pr
Charlie0113-T pr

45
third_party/pi/.github/ISSUE_TEMPLATE/bug.yml vendored Normal file
View File

@@ -0,0 +1,45 @@
name: Bug Report
description: Report something that's broken
labels: ["bug"]
body:
- type: markdown
attributes:
value: |
**Before you start:** Read [CONTRIBUTING.md](https://github.com/earendil-works/pi/blob/main/CONTRIBUTING.md).
New issues from new contributors are auto-closed by default. Maintainers review auto-closed issues daily. Issues that do not meet the quality bar in [CONTRIBUTING.md](https://github.com/earendil-works/pi/blob/main/CONTRIBUTING.md) will not be reopened or receive a reply.
Keep this short. If it doesn't fit on one screen, it's too long. Write in your own voice.
**Important:** before reporting an issue in core, please validate first with `pi -ne` that this is not caused by an extension you loaded.
- type: textarea
id: description
attributes:
label: What happened?
description: Be specific. Include error messages if any.
validations:
required: true
- type: textarea
id: repro
attributes:
label: Steps to reproduce
description: Minimal steps to trigger the bug.
validations:
required: true
- type: textarea
id: expected
attributes:
label: Expected behavior
validations:
required: false
- type: input
id: version
attributes:
label: Version
description: e.g. 0.49.0
validations:
required: false

5
third_party/pi/.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View File

@@ -0,0 +1,5 @@
blank_issues_enabled: false
contact_links:
- name: Questions
url: https://discord.com/invite/3cU7Bz4UPx
about: Ask questions on Discord instead of opening an issue

36
third_party/pi/.github/ISSUE_TEMPLATE/contribution.yml vendored Normal file
View File

@@ -0,0 +1,36 @@
name: Contribution Proposal
description: Propose a change or feature (required for new contributors before submitting a PR)
labels: []
body:
- type: markdown
attributes:
value: |
**Before you start:** Read [CONTRIBUTING.md](https://github.com/earendil-works/pi/blob/main/CONTRIBUTING.md).
New issues from new contributors are auto-closed by default. Maintainers review auto-closed issues daily. Issues that do not meet the quality bar in [CONTRIBUTING.md](https://github.com/earendil-works/pi/blob/main/CONTRIBUTING.md) will not be reopened or receive a reply.
Keep this short. If it doesn't fit on one screen, it's too long. Write in your own voice.
- type: textarea
id: what
attributes:
label: What do you want to change?
description: Be specific and concise.
validations:
required: true
- type: textarea
id: why
attributes:
label: Why?
description: What problem does this solve?
validations:
required: true
- type: textarea
id: how
attributes:
label: How? (optional)
description: Brief technical approach if you have one in mind.
validations:
required: false

49
third_party/pi/.github/ISSUE_TEMPLATE/package-report.yml vendored Normal file
View File

@@ -0,0 +1,49 @@
name: Package Report
description: Report a problematic Pi package listed on pi.dev
labels: ["package-report"]
body:
- type: markdown
attributes:
value: |
Use this form to report a package listed on pi.dev. For Pi core bugs, use the bug report template instead.
New issues from new contributors are auto-closed by default. Maintainers review auto-closed issues daily. Issues that do not meet the quality bar in [CONTRIBUTING.md](https://github.com/earendil-works/pi/blob/main/CONTRIBUTING.md) will not be reopened or receive a reply.
Keep this short. If it doesn't fit on one screen, it's too long. Write in your own voice.
- type: input
id: package-name
attributes:
label: Package name
description: The npm package name from pi.dev.
placeholder: "@scope/package"
validations:
required: true
- type: input
id: package-version
attributes:
label: Version
description: The package version shown on pi.dev.
placeholder: "0.1.0"
validations:
required: false
- type: dropdown
id: report-type
attributes:
label: What are you reporting?
options:
- Malicious or unsafe behavior
- Impersonation
- Trademark / TOS Violations
validations:
required: true
- type: textarea
id: details
attributes:
label: Details
description: Describe the concern and include links, logs, or screenshots if helpful.
validations:
required: true

223
third_party/pi/.github/workflows/approve-contributor.yml vendored Normal file
View File

@@ -0,0 +1,223 @@
name: Approve Contributor
on:
issue_comment:
types: [created]
jobs:
approve:
runs-on: ubuntu-latest
permissions:
contents: write
issues: write
pull-requests: write
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
- name: Update contributor approval
id: update
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const fs = require('fs');
const APPROVED_FILE = '.github/APPROVED_CONTRIBUTORS';
const VALID_CAPABILITIES = new Set(['issue', 'pr']);
const issueAuthor = context.payload.issue.user.login;
const commenter = context.payload.comment.user.login;
const commentBody = (context.payload.comment.body || '').trim();
const approvalAtStartPattern = /^[\s.]*(?:@[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?(?:\s*,\s*|[.:]\s*|\s+))*(lgtmi|lgtm)(?=$|[\s]|[^\p{L}\p{N}_\s])/iu;
const approvalAtEndPattern = /(?:^|[\s.])(lgtmi|lgtm)\s*(?:[^\p{L}\p{N}_\s])?\s*$/iu;
const approvalMatch = commentBody.match(approvalAtStartPattern) ?? commentBody.match(approvalAtEndPattern);
if (!approvalMatch) {
console.log('Comment does not start or end with lgtm or lgtmi');
core.setOutput('status', 'skipped');
return;
}
const targetCapability = approvalMatch[1].toLowerCase() === 'lgtmi' ? 'issue' : 'pr';
try {
const { data: permissionLevel } = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username: commenter,
});
if (!['admin', 'maintain', 'write'].includes(permissionLevel.permission)) {
console.log(`${commenter} does not have write access`);
core.setOutput('status', 'skipped');
return;
}
} catch {
console.log(`${commenter} does not have collaborator access`);
core.setOutput('status', 'skipped');
return;
}
function parseMentionedUsers(body) {
const users = [];
const seenUsers = new Set();
const mentionPattern = /(^|[^A-Za-z0-9_])@([A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?)(?![A-Za-z0-9-]|\/)/g;
for (const match of body.matchAll(mentionPattern)) {
const username = match[2];
const normalizedUser = username.toLowerCase();
if (seenUsers.has(normalizedUser)) {
continue;
}
seenUsers.add(normalizedUser);
users.push(username);
}
return users;
}
function parseApprovedUsers(content) {
const lines = content.split('\n');
const entries = [];
const users = new Map();
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) {
entries.push({ type: 'other', line });
continue;
}
const parts = trimmed.split(/\s+/);
if (parts.length !== 2) {
console.log(`Skipping malformed line: ${line}`);
entries.push({ type: 'other', line });
continue;
}
const [username, capability] = parts;
const normalizedCapability = capability.toLowerCase();
if (!VALID_CAPABILITIES.has(normalizedCapability)) {
console.log(`Skipping line with invalid capability: ${line}`);
entries.push({ type: 'other', line });
continue;
}
const normalizedUser = username.toLowerCase();
const entry = { type: 'user', username, normalizedUser, capability: normalizedCapability };
entries.push(entry);
users.set(normalizedUser, entry);
}
return { entries, users };
}
function stringifyApprovedUsers(entries) {
const normalizedEntries = [...entries];
while (normalizedEntries.length > 0) {
const lastEntry = normalizedEntries[normalizedEntries.length - 1];
if (lastEntry.type !== 'other' || lastEntry.line.trim() !== '') {
break;
}
normalizedEntries.pop();
}
return `${normalizedEntries
.map((entry) => (entry.type === 'user' ? `${entry.username} ${entry.capability}` : entry.line))
.join('\n')}\n`;
}
const content = fs.readFileSync(APPROVED_FILE, 'utf8');
const { entries, users } = parseApprovedUsers(content);
const mentionedUsers = parseMentionedUsers(commentBody);
const approvalTargets = mentionedUsers.length > 0 ? mentionedUsers : [issueAuthor];
const changedTargets = [];
const alreadyTargets = [];
for (const username of approvalTargets) {
const normalizedUser = username.toLowerCase();
const existingEntry = users.get(normalizedUser);
const existingCapability = existingEntry?.capability ?? null;
if (existingCapability === 'pr' || existingCapability === targetCapability) {
alreadyTargets.push(existingEntry?.username ?? username);
console.log(`${username} is already approved for ${existingCapability}`);
continue;
}
if (existingEntry) {
existingEntry.capability = targetCapability;
changedTargets.push(existingEntry.username);
} else {
const entry = { type: 'user', username, normalizedUser, capability: targetCapability };
entries.push(entry);
users.set(normalizedUser, entry);
changedTargets.push(username);
}
console.log(`Set ${username} capability to ${targetCapability}`);
}
core.setOutput('capability', targetCapability);
core.setOutput('changed_targets', JSON.stringify(changedTargets));
core.setOutput('already_targets', JSON.stringify(alreadyTargets));
if (changedTargets.length === 0) {
core.setOutput('status', 'already');
return;
}
fs.writeFileSync(APPROVED_FILE, stringifyApprovedUsers(entries));
core.setOutput('status', 'changed');
- name: Commit and push
if: steps.update.outputs.status == 'changed'
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git add .github/APPROVED_CONTRIBUTORS
git diff --staged --quiet || git commit -m "chore: approve contributors from issue #${{ github.event.issue.number }}"
git push
- name: Comment on issue
if: steps.update.outputs.status == 'changed' || steps.update.outputs.status == 'already'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
CAPABILITY: ${{ steps.update.outputs.capability }}
CHANGED_TARGETS: ${{ steps.update.outputs.changed_targets }}
ALREADY_TARGETS: ${{ steps.update.outputs.already_targets }}
with:
script: |
const capability = process.env.CAPABILITY;
const changedTargets = JSON.parse(process.env.CHANGED_TARGETS || '[]');
const alreadyTargets = JSON.parse(process.env.ALREADY_TARGETS || '[]');
const defaultBranch = context.payload.repository.default_branch;
const formatTargets = (targets) => targets.map((target) => `@${target}`).join(', ');
const bodyLines = [];
if (changedTargets.length > 0) {
if (capability === 'issue') {
bodyLines.push(`${formatTargets(changedTargets)} approved for issues. Future issues will not be auto-closed. PRs still require \`lgtm\` at the start of a maintainer reply (optionally after one or more \`@username\` mentions) or at the end.`);
} else {
bodyLines.push(`${formatTargets(changedTargets)} approved for issues and PRs. Future issues and PRs will not be auto-closed.`);
}
}
if (alreadyTargets.length > 0) {
const verb = alreadyTargets.length === 1 ? 'is' : 'are';
bodyLines.push(`${formatTargets(alreadyTargets)} ${verb} already approved.`);
}
bodyLines.push('', `See [CONTRIBUTING.md](https://github.com/${context.repo.owner}/${context.repo.repo}/blob/${defaultBranch}/CONTRIBUTING.md).`);
const body = bodyLines.join('\n');
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body,
});

444
third_party/pi/.github/workflows/build-binaries.yml vendored Normal file
View File

@@ -0,0 +1,444 @@
name: Build Binaries
on:
push:
tags:
- 'v*'
workflow_dispatch:
inputs:
tag:
description: 'Tag to build (e.g., v0.12.0)'
required: true
type: string
source_ref:
description: 'Source ref to build/publish (defaults to tag; use only for release recovery)'
required: false
type: string
permissions: {}
concurrency:
group: build-binaries-${{ github.event.inputs.tag || github.ref_name }}
cancel-in-progress: false
jobs:
# Keep the public GitHub Release publication last. Binary assets are staged in
# a draft release first; cleanup removes the draft if later publishing fails.
build:
runs-on: ubuntu-latest
permissions:
contents: read
env:
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
SOURCE_REF: ${{ github.event.inputs.source_ref || github.event.inputs.tag || github.ref_name }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ env.SOURCE_REF }}
persist-credentials: false
- name: Setup Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.3.14
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '22'
registry-url: 'https://registry.npmjs.org'
- name: Hydrate release model data
run: npm run hydrate:model-data
- name: Create source archive
run: |
set -euo pipefail
VERSION="${RELEASE_TAG#v}"
mkdir -p release-assets
./scripts/create-source-archive.sh \
--version "${VERSION}" \
--ref HEAD \
--out "release-assets/pi-${VERSION}-source.tar.gz"
- name: Build binaries from source archive
run: |
set -euo pipefail
VERSION="${RELEASE_TAG#v}"
build_root="$(mktemp -d)"
trap 'rm -rf "${build_root}"' EXIT
tar -xzf "release-assets/pi-${VERSION}-source.tar.gz" -C "${build_root}"
"${build_root}/pi-${VERSION}/scripts/build-binaries.sh" \
--offline-model-data \
--out "${GITHUB_WORKSPACE}/packages/coding-agent/binaries"
- name: Prepare GitHub release payload
run: |
set -euo pipefail
mkdir -p release-assets
VERSION="${RELEASE_TAG}"
VERSION="${VERSION#v}" # Remove 'v' prefix
node scripts/release-notes.mjs extract --version "${VERSION}" --tag "${RELEASE_TAG}" --out release-assets/RELEASE_NOTES.md
node scripts/generate-coding-agent-install-lock.mjs --check
cp packages/coding-agent/install-lock/package.json release-assets/pi-coding-agent-install-package.json
cp packages/coding-agent/install-lock/package-lock.json release-assets/pi-coding-agent-install-package-lock.json
cd packages/coding-agent/binaries
binary_assets=(
pi-darwin-arm64.tar.gz
pi-darwin-x64.tar.gz
pi-linux-x64.tar.gz
pi-linux-arm64.tar.gz
pi-windows-x64.zip
pi-windows-arm64.zip
)
for asset in "${binary_assets[@]}"; do
test -f "${asset}"
done
cp "${binary_assets[@]}" "${GITHUB_WORKSPACE}/release-assets/"
cd "${GITHUB_WORKSPACE}/release-assets"
source_asset="pi-${VERSION}-source.tar.gz"
release_assets=(
"${source_asset}"
pi-darwin-arm64.tar.gz
pi-darwin-x64.tar.gz
pi-linux-x64.tar.gz
pi-linux-arm64.tar.gz
pi-windows-x64.zip
pi-windows-arm64.zip
pi-coding-agent-install-package.json
pi-coding-agent-install-package-lock.json
)
sha256sum "${release_assets[@]}" > SHA256SUMS
- name: Upload GitHub release payload
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: release-assets-${{ env.RELEASE_TAG }}
path: release-assets/*
if-no-files-found: error
retention-days: 14
smoke-test-binaries:
runs-on: ${{ matrix.runner }}
needs: build
strategy:
fail-fast: false
matrix:
runner:
- ubuntu-latest
- macos-latest
- windows-latest
permissions:
actions: read
env:
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
steps:
- name: Download binary archives
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: release-assets-${{ env.RELEASE_TAG }}
path: release-assets
- name: Extract current-platform binary
id: binary
shell: bash
run: |
set -euo pipefail
runtime_platform="$(node -p '`${process.platform}-${process.arch}`')"
case "${runtime_platform}" in
darwin-arm64|darwin-x64|linux-arm64|linux-x64)
platform="${runtime_platform}"
archive="release-assets/pi-${platform}.tar.gz"
root="extracted/pi"
binary="${root}/pi"
;;
win32-arm64)
platform="windows-arm64"
archive="release-assets/pi-${platform}.zip"
root="extracted"
binary="${root}/pi.exe"
;;
win32-x64)
platform="windows-x64"
archive="release-assets/pi-${platform}.zip"
root="extracted"
binary="${root}/pi.exe"
;;
*)
echo "::error::Unsupported runner platform ${runtime_platform}"
exit 1
;;
esac
mkdir -p extracted
if [[ "${archive}" == *.zip ]]; then
export ARCHIVE_PATH="$(cygpath -w "${archive}")"
export DESTINATION_PATH="$(cygpath -w extracted)"
powershell.exe -NoProfile -NonInteractive -Command \
'Expand-Archive -LiteralPath $env:ARCHIVE_PATH -DestinationPath $env:DESTINATION_PATH -Force'
else
tar -xf "${archive}" -C extracted
fi
echo "root=${root}" >> "${GITHUB_OUTPUT}"
echo "binary=${binary}" >> "${GITHUB_OUTPUT}"
- name: Smoke-test binary
shell: bash
run: |
"${{ steps.binary.outputs.binary }}" --help
"${{ steps.binary.outputs.binary }}" --version
stage-github-release:
runs-on: ubuntu-latest
needs:
- build
- smoke-test-binaries
permissions:
actions: read
contents: write
env:
GH_REPO: ${{ github.repository }}
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
steps:
- name: Download GitHub release payload
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: release-assets-${{ env.RELEASE_TAG }}
path: release-assets
- name: Validate GitHub release payload
run: |
set -euo pipefail
cd release-assets
VERSION="${RELEASE_TAG#v}"
source_asset="pi-${VERSION}-source.tar.gz"
expected_assets=(
"${source_asset}"
pi-darwin-arm64.tar.gz
pi-darwin-x64.tar.gz
pi-linux-x64.tar.gz
pi-linux-arm64.tar.gz
pi-windows-x64.zip
pi-windows-arm64.zip
pi-coding-agent-install-package.json
pi-coding-agent-install-package-lock.json
SHA256SUMS
RELEASE_NOTES.md
)
for asset in "${expected_assets[@]}"; do
test -f "${asset}"
done
sha256sum -c SHA256SUMS
- name: Create draft GitHub Release and upload assets
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
cd release-assets
VERSION="${RELEASE_TAG#v}"
source_asset="pi-${VERSION}-source.tar.gz"
release_assets=(
"${source_asset}"
pi-darwin-arm64.tar.gz
pi-darwin-x64.tar.gz
pi-linux-x64.tar.gz
pi-linux-arm64.tar.gz
pi-windows-x64.zip
pi-windows-arm64.zip
pi-coding-agent-install-package.json
pi-coding-agent-install-package-lock.json
SHA256SUMS
)
existing_release="$(gh release view "${RELEASE_TAG}" --json isDraft --jq .isDraft 2>/dev/null || true)"
if [[ "${existing_release}" == "false" ]]; then
echo "::error::GitHub Release ${RELEASE_TAG} is already published. Refusing to mutate a public release."
exit 1
fi
if [[ "${existing_release}" == "true" ]]; then
gh release delete "${RELEASE_TAG}" --yes
fi
gh release create "${RELEASE_TAG}" \
--verify-tag \
--draft \
--title "${RELEASE_TAG}" \
--notes-file RELEASE_NOTES.md \
"${release_assets[@]}"
expected_asset_names="$(printf '%s\n' "${release_assets[@]}" | sort)"
actual_asset_names="$(gh release view "${RELEASE_TAG}" --json assets --jq '.assets[].name' | sort)"
if [[ "${actual_asset_names}" != "${expected_asset_names}" ]]; then
echo "::error::Draft GitHub Release asset set does not match expected files."
diff -u <(printf '%s\n' "${expected_asset_names}") <(printf '%s\n' "${actual_asset_names}") || true
exit 1
fi
publish-npm:
runs-on: ubuntu-latest
needs: stage-github-release
environment: npm-publish
permissions:
contents: read
id-token: write
env:
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
SOURCE_REF: ${{ github.event.inputs.source_ref || github.event.inputs.tag || github.ref_name }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ env.SOURCE_REF }}
persist-credentials: false
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '22'
registry-url: 'https://registry.npmjs.org'
cache: npm
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find ripgrep
sudo ln -s $(which fdfind) /usr/local/bin/fd
- name: Install dependencies
run: npm ci --ignore-scripts
- name: Build
run: npm run build
- name: Check
run: npm run check
- name: Test
run: npm test
- name: Smoke-test packed npm consumer
run: npm run check:package-install
- name: Upgrade npm for trusted publishing
run: |
npm install -g npm@11.16.0 --ignore-scripts
npm --version
- name: Publish npm packages
run: node scripts/publish.mjs
announce-pi-dev-release:
runs-on: ubuntu-latest
needs: publish-npm
environment: pi-model-upload
concurrency:
group: announce-pi-dev-release
cancel-in-progress: false
permissions:
contents: read
env:
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
SOURCE_REF: ${{ github.event.inputs.source_ref || github.event.inputs.tag || github.ref_name }}
AWS_ACCESS_KEY_ID: ${{ secrets.PI_ARTIFACTS_R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.PI_ARTIFACTS_R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_EC2_METADATA_DISABLED: 'true'
R2_ENDPOINT: https://67c0d357268b0fca6e0b465bb9d01b84.r2.cloudflarestorage.com
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ env.SOURCE_REF }}
persist-credentials: false
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '22'
- name: Verify AWS CLI
run: aws --version
- name: Announce verified release on pi.dev
run: |
node scripts/publish-release-announcement.mjs \
--version "${RELEASE_TAG#v}" \
--bucket pi-artifacts \
--endpoint "$R2_ENDPOINT" \
--installer-package-json packages/coding-agent/install-lock/package.json \
--installer-package-lock packages/coding-agent/install-lock/package-lock.json
publish-github-release:
runs-on: ubuntu-latest
needs:
- stage-github-release
- publish-npm
- announce-pi-dev-release
permissions:
contents: write
env:
GH_REPO: ${{ github.repository }}
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
steps:
- name: Publish staged GitHub Release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
existing_release="$(gh release view "${RELEASE_TAG}" --json isDraft --jq .isDraft 2>/dev/null || true)"
if [[ "${existing_release}" == "" ]]; then
echo "::error::Draft GitHub Release ${RELEASE_TAG} does not exist."
exit 1
fi
if [[ "${existing_release}" == "false" ]]; then
echo "::error::GitHub Release ${RELEASE_TAG} is already published."
exit 1
fi
gh release edit "${RELEASE_TAG}" --draft=false
cleanup-draft-github-release:
runs-on: ubuntu-latest
needs:
- build
- stage-github-release
- publish-npm
- announce-pi-dev-release
- publish-github-release
if: ${{ always() && needs.stage-github-release.result != 'skipped' && (needs.stage-github-release.result != 'success' || needs.publish-npm.result != 'success' || needs.announce-pi-dev-release.result != 'success' || needs.publish-github-release.result != 'success') }}
permissions:
contents: write
env:
GH_REPO: ${{ github.repository }}
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
steps:
- name: Delete draft GitHub Release after failure
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
existing_release="$(gh release view "${RELEASE_TAG}" --json isDraft --jq .isDraft 2>/dev/null || true)"
if [[ "${existing_release}" == "true" ]]; then
gh release delete "${RELEASE_TAG}" --yes
fi

42
third_party/pi/.github/workflows/ci.yml vendored Normal file
View File

@@ -0,0 +1,42 @@
name: CI
on:
push:
branches: [main]
pull_request:
branches: [main]
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
jobs:
build-check-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: npm
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find ripgrep
sudo ln -s $(which fdfind) /usr/local/bin/fd
- name: Install dependencies
run: npm ci --ignore-scripts
- name: Build
run: npm run build
- name: Check
run: npm run check
- name: Test
run: npm test

688
third_party/pi/.github/workflows/issue-analysis.yml vendored Normal file
View File

@@ -0,0 +1,688 @@
# Runs the repo's /is prompt against an issue when the `pi-analyze` label is added
# or when a staff member comments `@issuron analyze` anywhere on an issue.
#
# Comment triggers can include one `#run-on-*` tag anywhere in the text:
# @issuron analyze #run-on-linux -> ubuntu-latest (default)
# @issuron analyze #run-on-windows -> windows-latest
# @issuron analyze #run-on-mac -> macos-latest
#
# Label triggers always run on the default Linux runner. Runner selection is
# intentionally restricted to hardcoded aliases in the authorization step.
#
# Setup required before this works:
# 1. Create a `pi-analyze` GitHub environment on the repo and add a
# `PI_AUTH_JSON` secret containing the contents of a pi auth.json
# (~/.pi/agent/auth.json).
# 2. Create the `pi-analyze` label.
# 3. Add a repository secret `EARENDIL_ORG_READ_TOKEN` with permission to
# read `earendil-works` org membership. The authorization job uses it to
# verify that the label actor is an active member of `earendil-works/staff`.
# 4. Add an environment secret `PI_GIST_TOKEN` on `pi-analyze` with gist
# creation permission. The analysis job uses it to upload the exported
# session gist.
# 5. Add an environment secret `PI_AUTH_UPDATE_TOKEN` on `pi-analyze` with
# permission to update this repo's environment secrets. The analysis job
# uses it to write back refreshed `PI_AUTH_JSON` contents. Fine-grained
# tokens need repository permission `Environments: read and write` on this
# repo; classic tokens need the `repo` scope. The `Secrets` permission is
# not enough: it only covers repository-level Actions secrets, while every
# `/repos/{owner}/{repo}/environments/{env}/secrets/*` endpoint is governed
# by `Environments`. Without it `gh secret set` fails with "Resource not
# accessible by personal access token" and the stored credential goes stale
# until its refresh token is rejected.
#
# `PI_AUTH_JSON` must come from a login dedicated to this workflow. OpenAI Codex
# rotates refresh tokens on every refresh, so sharing one auth.json with a
# developer machine invalidates whichever copy refreshes second, and analysis
# then fails with "OAuth refresh failed for openai-codex".
#
# The selected runner must have Node.js support plus gh, fd, and ripgrep. GitHub
# hosted runners are bootstrapped below; future self-hosted aliases should have
# those dependencies preinstalled or installable by the setup steps.
#
# The session runs in a high-entropy checkout directory so the recorded cwd is
# a unique string. Import the session into a local checkout with the
# /ir extension command (.pi/extensions/import-repro.ts):
# pi "/ir <gist-id | gist-url | pi.dev/session URL>"
name: Issue Analysis
on:
issues:
types: [labeled]
issue_comment:
types: [created]
permissions:
contents: read
issues: write
concurrency:
group: issue-analysis-${{ github.event.issue.number }}
cancel-in-progress: false
jobs:
authorize:
runs-on: ubuntu-latest
outputs:
should_run: ${{ steps.verify.outputs.should_run }}
extra_instructions: ${{ steps.verify.outputs.extra_instructions }}
runs_on: ${{ steps.verify.outputs.runs_on }}
runner_os: ${{ steps.verify.outputs.runner_os }}
runner_profile: ${{ steps.verify.outputs.runner_profile }}
steps:
- name: Verify sender permission
id: verify
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
ORG_READ_TOKEN: ${{ secrets.EARENDIL_ORG_READ_TOKEN }}
with:
script: |
const ANALYZE_LABEL = 'pi-analyze';
const TRIGGER_RE = /@issuron\s+analyze\b/i;
const RUN_ON_TAG_RE = /#run-on-([a-z0-9][a-z0-9_-]*)\b/gi;
const RUNNER_PROFILES = {
linux: { runsOn: 'ubuntu-latest', os: 'linux' },
windows: { runsOn: 'windows-latest', os: 'windows' },
mac: { runsOn: 'macos-latest', os: 'macos' },
};
const RUN_ON_ALIASES = {
linux: 'linux',
ubuntu: 'linux',
'ubuntu-latest': 'linux',
windows: 'windows',
win: 'windows',
'windows-latest': 'windows',
mac: 'mac',
macos: 'mac',
darwin: 'mac',
'macos-latest': 'mac',
};
const username = context.payload.sender.login;
let extraInstructions = '';
let runnerProfile = 'linux';
core.setOutput('should_run', 'false');
core.setOutput('extra_instructions', '');
core.setOutput('runs_on', JSON.stringify(RUNNER_PROFILES.linux.runsOn));
core.setOutput('runner_os', RUNNER_PROFILES.linux.os);
core.setOutput('runner_profile', runnerProfile);
if (context.eventName === 'issues') {
if (context.payload.action !== 'labeled' || context.payload.label?.name !== ANALYZE_LABEL) {
console.log('Not a pi-analyze label event');
return;
}
} else if (context.eventName === 'issue_comment') {
if (context.payload.issue.pull_request) {
console.log('Ignoring pull request comment');
return;
}
const body = context.payload.comment.body || '';
if (!TRIGGER_RE.test(body)) {
console.log('Comment does not contain an @issuron analyze trigger');
return;
}
const resolvedProfiles = new Set();
const unknownTags = [];
for (const match of body.matchAll(RUN_ON_TAG_RE)) {
const tag = match[1].toLowerCase();
const resolved = RUN_ON_ALIASES[tag];
if (!resolved) {
unknownTags.push(tag);
} else {
resolvedProfiles.add(resolved);
}
}
if (unknownTags.length > 0) {
core.setFailed(`Unknown issue analysis runner tag(s): ${unknownTags.map((tag) => `#run-on-${tag}`).join(', ')}`);
return;
}
if (resolvedProfiles.size > 1) {
core.setFailed(
`Conflicting issue analysis runner tags: ${Array.from(resolvedProfiles)
.map((profile) => `#run-on-${profile}`)
.join(', ')}`,
);
return;
}
runnerProfile = Array.from(resolvedProfiles)[0] || 'linux';
extraInstructions = body.replace(TRIGGER_RE, ' ').replace(RUN_ON_TAG_RE, ' ').trim();
} else {
console.log(`Unsupported event: ${context.eventName}`);
return;
}
async function removeTriggerLabel() {
if (context.eventName !== 'issues') return;
try {
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
name: ANALYZE_LABEL,
});
} catch (error) {
if (error.status !== 404) throw error;
}
}
if (!process.env.ORG_READ_TOKEN) {
await removeTriggerLabel();
core.setFailed('EARENDIL_ORG_READ_TOKEN is not configured; refusing to run issue analysis.');
return;
}
try {
const response = await fetch(
`https://api.github.com/orgs/earendil-works/teams/staff/memberships/${encodeURIComponent(username)}`,
{
headers: {
Accept: 'application/vnd.github+json',
Authorization: `Bearer ${process.env.ORG_READ_TOKEN}`,
'X-GitHub-Api-Version': '2022-11-28',
},
},
);
if (response.status === 404) {
await removeTriggerLabel();
core.setFailed(`@${username} is not an active earendil-works/staff member.`);
return;
}
if (!response.ok) {
const body = await response.text();
await removeTriggerLabel();
core.setFailed(
`Could not verify earendil-works/staff membership for @${username}: HTTP ${response.status} ${body}`,
);
return;
}
const membership = await response.json();
if (membership.state !== 'active') {
await removeTriggerLabel();
core.setFailed(`@${username} is not an active earendil-works/staff member.`);
return;
}
console.log(`earendil-works/staff membership for @${username}: ${membership.state}`);
} catch (error) {
await removeTriggerLabel();
core.setFailed(
`Could not verify earendil-works/staff membership for @${username}: ${
error instanceof Error ? error.message : String(error)
}`,
);
return;
}
const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username,
});
if (!['admin', 'write'].includes(data.permission)) {
await removeTriggerLabel();
core.setFailed(
`@${username} has '${data.permission}' permission; write or admin is required to trigger issue analysis.`,
);
return;
}
if (context.eventName === 'issue_comment') {
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: [ANALYZE_LABEL],
});
}
const profile = RUNNER_PROFILES[runnerProfile];
console.log(`Selected issue analysis runner profile: ${runnerProfile} (${JSON.stringify(profile.runsOn)})`);
core.setOutput('should_run', 'true');
core.setOutput('extra_instructions', extraInstructions);
core.setOutput('runs_on', JSON.stringify(profile.runsOn));
core.setOutput('runner_os', profile.os);
core.setOutput('runner_profile', runnerProfile);
analyze:
needs: authorize
if: needs.authorize.outputs.should_run == 'true'
runs-on: ${{ fromJSON(needs.authorize.outputs.runs_on) }}
environment: pi-analyze
timeout-minutes: 45
concurrency:
group: issue-analysis-pi-auth
cancel-in-progress: false
env:
ISSUE_ANALYSIS_MODEL: openai-codex/gpt-5.5
ISSUE_ANALYSIS_THINKING: high
steps:
- name: Create high-entropy working directory name
id: workdir
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const crypto = require('crypto');
core.setOutput('name', `pi-ci-${crypto.randomBytes(16).toString('hex')}`);
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
path: ${{ steps.workdir.outputs.name }}
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: npm
cache-dependency-path: ${{ steps.workdir.outputs.name }}/package-lock.json
- name: Install system dependencies (Linux)
if: needs.authorize.outputs.runner_os == 'linux'
run: |
sudo apt-get update
sudo apt-get install -y fd-find ripgrep
sudo ln -sf "$(which fdfind)" /usr/local/bin/fd
- name: Install system dependencies (macOS)
if: needs.authorize.outputs.runner_os == 'macos'
run: |
if ! command -v fd >/dev/null 2>&1; then
brew install fd
fi
if ! command -v rg >/dev/null 2>&1; then
brew install ripgrep
fi
- name: Install system dependencies (Windows)
if: needs.authorize.outputs.runner_os == 'windows'
shell: pwsh
run: |
$packages = @()
if (-not (Get-Command fd -ErrorAction SilentlyContinue)) {
$packages += "fd"
}
if (-not (Get-Command rg -ErrorAction SilentlyContinue)) {
$packages += "ripgrep"
}
if ($packages.Count -gt 0) {
if (-not (Get-Command choco -ErrorAction SilentlyContinue)) {
throw "fd and ripgrep must be installed on Windows runners, or Chocolatey must be available to install them."
}
choco install $packages -y --no-progress
}
fd --version
rg --version
- name: Install dependencies
working-directory: ${{ steps.workdir.outputs.name }}
run: npm ci --ignore-scripts
- name: Build
working-directory: ${{ steps.workdir.outputs.name }}
run: npm run build
- name: Write auth.json
id: write_auth
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
PI_AUTH_JSON: ${{ secrets.PI_AUTH_JSON }}
with:
script: |
const fs = require('fs');
const path = require('path');
const authJson = process.env.PI_AUTH_JSON;
if (!authJson) {
throw new Error('PI_AUTH_JSON secret is not configured for the pi-analyze environment');
}
const agentDir = path.join(process.env.RUNNER_TEMP, 'pi-agent');
fs.mkdirSync(agentDir, { recursive: true });
const authPath = path.join(agentDir, 'auth.json');
fs.writeFileSync(authPath, authJson, { mode: 0o600 });
if (process.platform !== 'win32') {
fs.chmodSync(authPath, 0o600);
}
- name: Run pi /is
id: run_pi
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
PI_CODING_AGENT_DIR: ${{ runner.temp }}/pi-agent
GH_TOKEN: ${{ github.token }}
ISSUE_URL: ${{ github.event.issue.html_url }}
EXTRA_INSTRUCTIONS: ${{ needs.authorize.outputs.extra_instructions }}
WORKDIR: ${{ steps.workdir.outputs.name }}
with:
script: |
const fs = require('fs');
const path = require('path');
const { spawn } = require('child_process');
const workdir = path.join(process.env.GITHUB_WORKSPACE, process.env.WORKDIR);
const outDir = path.join(process.env.RUNNER_TEMP, 'pi-out');
const sessionDir = path.join(outDir, 'session');
fs.mkdirSync(sessionDir, { recursive: true });
let prompt = `/is ${process.env.ISSUE_URL}`;
if (process.env.EXTRA_INSTRUCTIONS) {
prompt += '\n\nAdditional instructions from @issuron analyze comment:\n';
prompt += process.env.EXTRA_INSTRUCTIONS;
}
const outputPath = path.join(outDir, 'output.md');
const output = fs.createWriteStream(outputPath);
const args = [
'packages/coding-agent/src/cli.ts',
'-p',
'--approve',
'--session-dir',
sessionDir,
'--model',
process.env.ISSUE_ANALYSIS_MODEL,
'--thinking',
process.env.ISSUE_ANALYSIS_THINKING,
prompt,
];
const exitCode = await new Promise((resolve, reject) => {
const child = spawn('node', args, {
cwd: workdir,
env: process.env,
stdio: ['ignore', 'pipe', 'pipe'],
});
child.stdout.on('data', (chunk) => {
process.stdout.write(chunk);
output.write(chunk);
});
child.stderr.on('data', (chunk) => {
process.stderr.write(chunk);
});
child.on('error', reject);
child.on('close', resolve);
});
await new Promise((resolve) => output.end(resolve));
if (exitCode !== 0) {
throw new Error(`pi /is failed with exit code ${exitCode}`);
}
- name: Persist refreshed auth.json
if: always() && steps.write_auth.outcome == 'success'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
GH_TOKEN: ${{ secrets.PI_AUTH_UPDATE_TOKEN }}
PI_AUTH_JSON: ${{ secrets.PI_AUTH_JSON }}
PI_CODING_AGENT_DIR: ${{ runner.temp }}/pi-agent
with:
script: |
const fs = require('fs');
const path = require('path');
const { spawn } = require('child_process');
if (!process.env.GH_TOKEN) {
throw new Error('PI_AUTH_UPDATE_TOKEN is not configured for the pi-analyze environment');
}
const authPath = path.join(process.env.PI_CODING_AGENT_DIR, 'auth.json');
if (!fs.existsSync(authPath)) {
core.warning('auth.json was not created; skipping auth persistence');
return;
}
const authJson = fs.readFileSync(authPath, 'utf8');
let parsed;
try {
parsed = JSON.parse(authJson);
} catch (error) {
throw new Error(`Refusing to persist malformed auth.json: ${error instanceof Error ? error.message : String(error)}`);
}
const codexAuth = parsed['openai-codex'];
if (codexAuth?.type !== 'oauth' || typeof codexAuth.refresh !== 'string' || codexAuth.refresh.length === 0) {
throw new Error('Refusing to persist auth.json without openai-codex OAuth refresh credentials');
}
if (authJson === process.env.PI_AUTH_JSON) {
console.log('auth.json is unchanged; skipping secret update');
return;
}
await new Promise((resolve, reject) => {
const child = spawn(
'gh',
['secret', 'set', 'PI_AUTH_JSON', '--env', 'pi-analyze', '--repo', process.env.GITHUB_REPOSITORY],
{ env: process.env, stdio: ['pipe', 'inherit', 'pipe'] },
);
const stderrChunks = [];
child.stderr.on('data', (chunk) => {
stderrChunks.push(chunk);
process.stderr.write(chunk);
});
child.on('error', reject);
child.on('close', (code) => {
if (code === 0) {
resolve();
return;
}
const details = Buffer.concat(stderrChunks).toString().trim();
reject(
new Error(
`gh secret set failed with exit code ${code}${details ? `: ${details}` : ''}. ` +
'PI_AUTH_UPDATE_TOKEN needs repository permission "Environments: read and write" on this repo.',
),
);
});
child.stdin.end(authJson);
});
- name: Export session files
id: export_session_files
if: always()
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
PI_CODING_AGENT_DIR: ${{ runner.temp }}/pi-agent
WORKDIR: ${{ steps.workdir.outputs.name }}
with:
script: |
const fs = require('fs');
const path = require('path');
const { spawn } = require('child_process');
function findFirstJsonl(dir) {
if (!fs.existsSync(dir)) return undefined;
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const entryPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
const nested = findFirstJsonl(entryPath);
if (nested) return nested;
} else if (entry.isFile() && entry.name.endsWith('.jsonl')) {
return entryPath;
}
}
return undefined;
}
const outDir = path.join(process.env.RUNNER_TEMP, 'pi-out');
const sessionFile = findFirstJsonl(path.join(outDir, 'session'));
if (!sessionFile) {
throw new Error('No session jsonl file found');
}
const sessionJsonl = path.join(outDir, 'session.jsonl');
const sessionHtml = path.join(outDir, 'session.html');
fs.copyFileSync(sessionFile, sessionJsonl);
const workdir = path.join(process.env.GITHUB_WORKSPACE, process.env.WORKDIR);
const exitCode = await new Promise((resolve, reject) => {
const child = spawn(
'node',
['packages/coding-agent/src/cli.ts', '--no-extensions', '--export', sessionJsonl, sessionHtml],
{ cwd: workdir, env: process.env, stdio: 'inherit' },
);
child.on('error', reject);
child.on('close', resolve);
});
if (exitCode !== 0) {
throw new Error(`session export failed with exit code ${exitCode}`);
}
- name: Upload session gist
id: gist
if: always() && steps.export_session_files.outcome == 'success'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
PI_GIST_TOKEN: ${{ secrets.PI_GIST_TOKEN }}
with:
github-token: ${{ secrets.PI_GIST_TOKEN }}
script: |
const fs = require('fs');
const path = require('path');
if (!process.env.PI_GIST_TOKEN) {
throw new Error('PI_GIST_TOKEN is not configured');
}
const outDir = path.join(process.env.RUNNER_TEMP, 'pi-out');
const files = {};
for (const filename of ['session.html', 'session.jsonl']) {
files[filename] = { content: fs.readFileSync(path.join(outDir, filename), 'utf8') };
}
const response = await github.rest.gists.create({
public: false,
files,
});
const gistUrl = response.data.html_url;
const gistId = response.data.id;
core.setOutput('url', gistUrl);
core.setOutput('id', gistId);
core.setOutput('share_url', `https://pi.dev/session/#${gistId}`);
- name: Comment with session import instructions
if: always() && steps.run_pi.outcome == 'success' && steps.gist.outcome == 'success'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
GIST_URL: ${{ steps.gist.outputs.url }}
GIST_ID: ${{ steps.gist.outputs.id }}
SHARE_URL: ${{ steps.gist.outputs.share_url }}
SESSION_JSONL: ${{ runner.temp }}/pi-out/session.jsonl
with:
script: |
const fs = require('fs');
function extractLastAgentMessage(sessionPath) {
const lines = fs.readFileSync(sessionPath, 'utf8').split(/\r?\n/).filter(Boolean);
let lastText = '';
for (const line of lines) {
let entry;
try {
entry = JSON.parse(line);
} catch {
continue;
}
if (entry.type !== 'message' || entry.message?.role !== 'assistant') continue;
const content = entry.message.content;
const parts = [];
if (typeof content === 'string') {
parts.push(content);
} else if (Array.isArray(content)) {
for (const block of content) {
if (block?.type === 'text' && typeof block.text === 'string') {
parts.push(block.text);
}
}
}
const text = parts.join('\n\n').trim();
if (text) lastText = text;
}
if (!lastText) return '_No assistant output found._';
const maxLength = 55000;
if (lastText.length <= maxLength) return lastText;
return `${lastText.slice(0, maxLength)}\n\n_[truncated]_`;
}
const gistUrl = process.env.GIST_URL;
const gistId = process.env.GIST_ID;
const shareUrl = process.env.SHARE_URL;
const lastAgentMessage = extractLastAgentMessage(process.env.SESSION_JSONL);
const body = [
'Pi issue analysis finished.',
'',
`Share URL: ${shareUrl}`,
`Gist: ${gistUrl}`,
'',
'Continue locally from a checkout with:',
'',
'```sh',
`pi "/ir ${gistId}"`,
'```',
'',
'<details>',
'<summary>Agent analysis summary</summary>',
'',
lastAgentMessage,
'',
'</details>',
].join('\n');
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body,
});
- name: Comment on analysis failure
if: always() && steps.run_pi.outcome == 'failure'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
GIST_URL: ${{ steps.gist.outputs.url }}
SHARE_URL: ${{ steps.gist.outputs.share_url }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
script: |
const lines = ['Pi issue analysis failed; no analysis was produced.', '', `Run: ${process.env.RUN_URL}`];
if (process.env.SHARE_URL) {
lines.push(`Partial session: ${process.env.SHARE_URL}`);
}
if (process.env.GIST_URL) {
lines.push(`Gist: ${process.env.GIST_URL}`);
}
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: lines.join('\n'),
});
- name: Remove trigger label
if: always()
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
try {
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
name: 'pi-analyze',
});
} catch (error) {
if (error.status !== 404) throw error;
}

129
third_party/pi/.github/workflows/issue-gate.yml vendored Normal file
View File

@@ -0,0 +1,129 @@
name: Issue Gate
on:
issues:
types: [opened]
jobs:
check-contributor:
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
steps:
- name: Check issue author
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const APPROVED_FILE = '.github/APPROVED_CONTRIBUTORS';
const VALID_CAPABILITIES = new Set(['issue', 'pr']);
const TRUSTED_BOT_AUTHORS = new Set(['dependabot[bot]', 'sentry[bot]', 'claude[bot]']);
const issueAuthor = context.payload.issue.user.login;
const defaultBranch = context.payload.repository.default_branch;
const isBotAuthor = issueAuthor.endsWith('[bot]');
if (TRUSTED_BOT_AUTHORS.has(issueAuthor)) {
console.log(`Skipping trusted bot: ${issueAuthor}`);
return;
}
async function getPermission(username) {
try {
const { data: permissionLevel } = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username,
});
return permissionLevel.permission;
} catch {
return null;
}
}
async function getTextFile(path) {
const { data: fileContent } = await github.rest.repos.getContent({
owner: context.repo.owner,
repo: context.repo.repo,
path,
ref: defaultBranch,
});
if (!('content' in fileContent) || typeof fileContent.content !== 'string') {
throw new Error(`Expected file content for ${path}`);
}
return Buffer.from(fileContent.content, 'base64').toString('utf8');
}
function parseApprovedUsers(content) {
const users = new Map();
for (const rawLine of content.split('\n')) {
const line = rawLine.trim();
if (!line || line.startsWith('#')) continue;
const parts = line.split(/\s+/);
if (parts.length !== 2) {
console.log(`Skipping malformed line: ${rawLine}`);
continue;
}
const [username, capability] = parts;
const normalizedCapability = capability.toLowerCase();
if (!VALID_CAPABILITIES.has(normalizedCapability)) {
console.log(`Skipping line with invalid capability: ${rawLine}`);
continue;
}
users.set(username.toLowerCase(), normalizedCapability);
}
return users;
}
const permission = await getPermission(issueAuthor);
if (!isBotAuthor && ['admin', 'maintain', 'write'].includes(permission)) {
console.log(`${issueAuthor} is a collaborator with ${permission} access`);
return;
}
const approvedContent = await getTextFile(APPROVED_FILE);
const approvedUsers = parseApprovedUsers(approvedContent);
const capability = approvedUsers.get(issueAuthor.toLowerCase());
if (!isBotAuthor && (capability === 'issue' || capability === 'pr')) {
console.log(`${issueAuthor} is approved for ${capability}`);
return;
}
const message = [
'This issue was auto-closed. All issues from new contributors are auto-closed by default.',
'',
`Maintainers review auto-closed issues daily and reopen worthwhile ones. Issues that do not meet the quality bar in [CONTRIBUTING.md](https://github.com/${context.repo.owner}/${context.repo.repo}/blob/${defaultBranch}/CONTRIBUTING.md) will not be reopened or receive a reply.`,
'',
'If a maintainer replies `lgtmi` on one of your issues, your future issues will stay open. If a maintainer replies `lgtm`, your future issues and PRs will stay open. The command must be at the start of the reply (optionally after one or more `@username` mentions) or at the end.',
'',
`See [CONTRIBUTING.md](https://github.com/${context.repo.owner}/${context.repo.repo}/blob/${defaultBranch}/CONTRIBUTING.md).`,
].join('\n');
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: message,
});
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
labels: ['untriaged'],
});
await github.rest.issues.update({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
state: 'closed',
state_reason: 'not_planned',
});

142
third_party/pi/.github/workflows/issue-triage-labels.yml vendored Normal file
View File

@@ -0,0 +1,142 @@
name: Issue Triage Labels
on:
issues:
types: [reopened, labeled]
jobs:
update-labels:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Update triage labels
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const UNTRIAGED_LABEL = 'untriaged';
const NO_ACTION_LABEL = 'no-action';
const LAST_READ_LABEL = 'last-read';
const TO_DISCUSS_LABEL = 'to-discuss';
const INPROGRESS_LABEL = 'inprogress';
function issueHasLabel(issue, labelName) {
return (issue.labels ?? []).some((label) => label.name === labelName);
}
async function removeLabelIfPresent(issueNumber, issue, labelName) {
if (!issueHasLabel(issue, labelName)) {
console.log(`Issue #${issueNumber} does not have ${labelName}`);
return;
}
try {
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issueNumber,
name: labelName,
});
console.log(`Removed ${labelName} from #${issueNumber}`);
} catch (error) {
if (error.status === 404) {
console.log(`Label ${labelName} was already absent from #${issueNumber}`);
return;
}
throw error;
}
}
if (context.payload.action === 'reopened') {
await removeLabelIfPresent(context.issue.number, context.payload.issue, UNTRIAGED_LABEL);
await removeLabelIfPresent(context.issue.number, context.payload.issue, NO_ACTION_LABEL);
return;
}
if (context.payload.action === 'labeled' && context.payload.label?.name === NO_ACTION_LABEL) {
await removeLabelIfPresent(context.issue.number, context.payload.issue, UNTRIAGED_LABEL);
return;
}
if (context.payload.action !== 'labeled' || context.payload.label?.name !== LAST_READ_LABEL) {
console.log('Not a last-read label event');
return;
}
const currentIssueNumber = context.issue.number;
const lastReadIssues = await github.paginate(github.rest.issues.listForRepo, {
owner: context.repo.owner,
repo: context.repo.repo,
state: 'all',
labels: LAST_READ_LABEL,
per_page: 100,
});
const previousIssueNumbers = lastReadIssues
.filter((issue) => !issue.pull_request)
.map((issue) => issue.number)
.filter((issueNumber) => issueNumber !== currentIssueNumber);
if (previousIssueNumbers.length === 0) {
console.log('No previous last-read issue found');
return;
}
const previousIssueNumber = Math.max(...previousIssueNumbers);
if (currentIssueNumber <= previousIssueNumber) {
console.log(
`Last-read was added to old issue #${currentIssueNumber}; latest last-read is #${previousIssueNumber}`,
);
return;
}
const untriagedIssues = await github.paginate(github.rest.issues.listForRepo, {
owner: context.repo.owner,
repo: context.repo.repo,
state: 'all',
labels: UNTRIAGED_LABEL,
per_page: 100,
});
const issuesToMark = untriagedIssues
.filter((issue) => !issue.pull_request)
.filter((issue) => issue.number >= previousIssueNumber && issue.number <= currentIssueNumber)
.sort((a, b) => a.number - b.number);
if (issuesToMark.length === 0) {
console.log(`No untriaged issues found from #${previousIssueNumber} to #${currentIssueNumber}`);
return;
}
for (const issue of issuesToMark) {
if (issueHasLabel(issue, TO_DISCUSS_LABEL)) {
console.log(`Skipped ${NO_ACTION_LABEL} for #${issue.number} because it has ${TO_DISCUSS_LABEL}`);
} else {
await github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue.number,
labels: [NO_ACTION_LABEL],
});
console.log(`Added ${NO_ACTION_LABEL} to #${issue.number}`);
}
await github.rest.issues.update({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue.number,
state: 'closed',
state_reason: 'not_planned',
});
console.log(`Closed #${issue.number} as not planned`);
await removeLabelIfPresent(issue.number, issue, INPROGRESS_LABEL);
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue.number,
name: UNTRIAGED_LABEL,
});
console.log(`Removed ${UNTRIAGED_LABEL} from #${issue.number}`);
}

31
third_party/pi/.github/workflows/npm-audit.yml vendored Normal file
View File

@@ -0,0 +1,31 @@
name: npm audit
on:
schedule:
- cron: '37 7 * * *'
workflow_dispatch:
permissions:
contents: read
jobs:
audit:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: npm
- name: Install dependencies without lifecycle scripts
run: npm ci --ignore-scripts --no-audit --no-fund
- name: Audit production vulnerabilities
run: npm audit --omit=dev --audit-level=moderate
- name: Verify registry signatures
run: npm audit signatures --omit=dev

128
third_party/pi/.github/workflows/pr-gate.yml vendored Normal file
View File

@@ -0,0 +1,128 @@
name: PR Gate
on:
pull_request_target:
types: [opened]
jobs:
check-contributor:
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
pull-requests: write
steps:
- name: Check if contributor is approved
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const APPROVED_FILE = '.github/APPROVED_CONTRIBUTORS';
const VALID_CAPABILITIES = new Set(['issue', 'pr']);
const TRUSTED_BOT_AUTHORS = new Set(['dependabot[bot]', 'sentry[bot]', 'claude[bot]']);
const prAuthor = context.payload.pull_request.user.login;
const defaultBranch = context.payload.repository.default_branch;
const isBotAuthor = prAuthor.endsWith('[bot]');
if (TRUSTED_BOT_AUTHORS.has(prAuthor)) {
console.log(`Skipping trusted bot: ${prAuthor}`);
return;
}
async function getPermission(username) {
try {
const { data: permissionLevel } = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username,
});
return permissionLevel.permission;
} catch {
return null;
}
}
async function getTextFile(path) {
const { data: fileContent } = await github.rest.repos.getContent({
owner: context.repo.owner,
repo: context.repo.repo,
path,
ref: defaultBranch,
});
if (!('content' in fileContent) || typeof fileContent.content !== 'string') {
throw new Error(`Expected file content for ${path}`);
}
return Buffer.from(fileContent.content, 'base64').toString('utf8');
}
function parseApprovedUsers(content) {
const users = new Map();
for (const rawLine of content.split('\n')) {
const line = rawLine.trim();
if (!line || line.startsWith('#')) continue;
const parts = line.split(/\s+/);
if (parts.length !== 2) {
console.log(`Skipping malformed line: ${rawLine}`);
continue;
}
const [username, capability] = parts;
const normalizedCapability = capability.toLowerCase();
if (!VALID_CAPABILITIES.has(normalizedCapability)) {
console.log(`Skipping line with invalid capability: ${rawLine}`);
continue;
}
users.set(username.toLowerCase(), normalizedCapability);
}
return users;
}
async function closePullRequest(message) {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.payload.pull_request.number,
body: message,
});
await github.rest.pulls.update({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.payload.pull_request.number,
state: 'closed',
});
}
const permission = await getPermission(prAuthor);
if (!isBotAuthor && ['admin', 'maintain', 'write'].includes(permission)) {
console.log(`${prAuthor} is a collaborator with ${permission} access`);
return;
}
const approvedContent = await getTextFile(APPROVED_FILE);
const approvedUsers = parseApprovedUsers(approvedContent);
const capability = approvedUsers.get(prAuthor.toLowerCase());
if (!isBotAuthor && capability === 'pr') {
console.log(`${prAuthor} is approved for PRs`);
return;
}
console.log(`${prAuthor} is not approved, closing PR`);
const message = [
'This PR was auto-closed. Only contributors approved with `lgtm` can open PRs. Open an issue first and ask a maintainer for approval.',
'',
`Maintainers review auto-closed issues daily. Issues that do not meet the quality bar in [CONTRIBUTING.md](https://github.com/${context.repo.owner}/${context.repo.repo}/blob/${defaultBranch}/CONTRIBUTING.md) will not be reopened or receive a reply.`,
'',
'If a maintainer replies `lgtmi`, your future issues will stay open. If a maintainer replies `lgtm`, your future issues and PRs will stay open. The command must be at the start of the reply (optionally after one or more `@username` mentions) or at the end.',
'',
`See [CONTRIBUTING.md](https://github.com/${context.repo.owner}/${context.repo.repo}/blob/${defaultBranch}/CONTRIBUTING.md).`,
].join('\n');
await closePullRequest(message);

146
third_party/pi/.github/workflows/publish-model-catalog.yml vendored Normal file
View File

@@ -0,0 +1,146 @@
name: Publish Model Catalog
on:
workflow_run:
workflows:
- CI
types:
- completed
pull_request:
paths:
- '.github/workflows/publish-model-catalog.yml'
- '.gitignore'
- 'package.json'
- 'packages/ai/**'
- 'scripts/publish-model-catalog.mjs'
# GitHub schedules use UTC. Run hourly candidates across the CET/CEST
# boundaries; the publish job only uploads at 10:17, 12:17, and 14:17
# Europe/Vienna time.
schedule:
- cron: '17 8-13 * * 1-5'
workflow_dispatch:
inputs:
source_ref:
description: 'Commit, branch, or tag to generate from'
required: false
default: 'main'
type: string
publish:
description: 'Upload the generated catalog to production R2'
required: true
default: false
type: boolean
permissions:
contents: read
jobs:
generate:
if: ${{ github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main') }}
runs-on: ubuntu-latest
env:
SOURCE_REF: ${{ github.event.workflow_run.head_sha || inputs.source_ref || github.sha }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ env.SOURCE_REF }}
persist-credentials: false
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '22'
cache: npm
- name: Install dependencies
run: npm ci --ignore-scripts
- name: Generate model catalog JSON
run: npm run generate:model-catalog
- name: Validate model catalog JSON
run: npm run check:model-catalog
- name: Upload model catalog JSON
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: model-catalog-json
path: .artifacts/model-catalog
if-no-files-found: error
retention-days: 14
publish:
if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_run' || (github.event_name == 'workflow_dispatch' && inputs.publish) }}
needs: generate
runs-on: ubuntu-latest
environment: pi-model-upload
concurrency:
group: publish-model-catalog-r2
cancel-in-progress: true
env:
SOURCE_REF: ${{ github.event.workflow_run.head_sha || inputs.source_ref || github.sha }}
AWS_ACCESS_KEY_ID: ${{ secrets.PI_ARTIFACTS_R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.PI_ARTIFACTS_R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_EC2_METADATA_DISABLED: 'true'
R2_ENDPOINT: https://67c0d357268b0fca6e0b465bb9d01b84.r2.cloudflarestorage.com
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ env.SOURCE_REF }}
persist-credentials: false
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '22'
- name: Download model catalog JSON
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: model-catalog-json
path: .artifacts/model-catalog
- name: Verify AWS CLI
run: aws --version
- name: Check publication window
id: publication-window
env:
EVENT_NAME: ${{ github.event_name }}
MANUAL_PUBLISH: ${{ inputs.publish || false }}
run: |
set -euo pipefail
local_day="$(TZ=Europe/Vienna date +%u)"
local_hour_text="$(TZ=Europe/Vienna date +%H)"
local_hour="$((10#$local_hour_text))"
local_time="$(TZ=Europe/Vienna date '+%Y-%m-%d %H:%M:%S %Z')"
allowed=false
reason="outside the Monday-Friday 10:00-15:00 Europe/Vienna publication window"
if [[ "$EVENT_NAME" == "workflow_dispatch" && "$MANUAL_PUBLISH" == "true" ]]; then
allowed=true
reason="manual publication"
elif (( local_day <= 5 && local_hour >= 10 && local_hour < 15 )); then
if [[ "$EVENT_NAME" != "schedule" ]] || (( (local_hour - 10) % 2 == 0 )); then
allowed=true
reason="business-hours publication"
else
reason="not a scheduled 10:17, 12:17, or 14:17 Europe/Vienna publication"
fi
fi
echo "allowed=$allowed" >> "$GITHUB_OUTPUT"
echo "R2 publication allowed: $allowed ($reason; local time: $local_time)"
- name: Publish model catalog to R2
if: steps.publication-window.outputs.allowed == 'true'
run: |
node scripts/publish-model-catalog.mjs \
--input .artifacts/model-catalog \
--bucket pi-artifacts \
--endpoint "$R2_ENDPOINT" \
--source-commit "$(git rev-parse HEAD)"

31
third_party/pi/.github/workflows/remove-inprogress-on-close.yml vendored Normal file
View File

@@ -0,0 +1,31 @@
name: Remove In Progress Label On Close
on:
issues:
types: [closed]
jobs:
remove-label:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Remove inprogress label
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const labelName = 'inprogress';
const labels = context.payload.issue.labels ?? [];
const hasLabel = labels.some((label) => label.name === labelName);
if (!hasLabel) {
console.log(`Issue does not have ${labelName} label`);
return;
}
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
name: labelName,
});

45
third_party/pi/.gitignore vendored Normal file
View File

@@ -0,0 +1,45 @@
node_modules/
dist/
.artifacts/
*.log
.DS_Store
*.tsbuildinfo
# packages/*/node_modules/
packages/*/dist/
packages/*/dist-chrome/
packages/*/dist-firefox/
packages/ai/src/providers/data/
packages/ai/src/providers/.model-generation-*/
*.cpuprofile
# Environment
.env
# Editor files
.vscode/
.zed/
.idea/
.claude/
*.swp
*.swo
*~
# Package specific
.npm/
coverage/
.nyc_output/
.pi_config/
tui-debug.log
compaction-results/
.opencode/
syntax.jsonl
out.jsonl
pi-*.html
out.html
packages/coding-agent/binaries/
todo.md
plans/
.pi/hf-sessions/
.pi/hf-sessions-backup/
collect.sh
packages/agent/docs/extensions/pi-server-artifact/index.md

45
third_party/pi/.husky/pre-commit vendored Executable file
View File

@@ -0,0 +1,45 @@
#!/bin/sh
# Get list of staged files before running check
STAGED_FILES=$(git diff --cached --name-only)
node scripts/check-lockfile-commit.mjs
if [ $? -ne 0 ]; then
exit 1
fi
# Run the check script (formatting, linting, and type checking)
echo "Running formatting, linting, and type checking..."
npm run check
if [ $? -ne 0 ]; then
echo "❌ Checks failed. Please fix the errors before committing."
exit 1
fi
RUN_BROWSER_SMOKE=0
for file in $STAGED_FILES; do
case "$file" in
packages/ai/*|packages/web-ui/*|package.json|package-lock.json)
RUN_BROWSER_SMOKE=1
break
;;
esac
done
if [ $RUN_BROWSER_SMOKE -eq 1 ]; then
echo "Running browser smoke check..."
npm run check:browser-smoke
if [ $? -ne 0 ]; then
echo "❌ Browser smoke check failed."
exit 1
fi
fi
# Restage files that were previously staged and may have been modified by formatting
for file in $STAGED_FILES; do
if [ -f "$file" ]; then
git add "$file"
fi
done
echo "✅ All pre-commit checks passed!"

2
third_party/pi/.npmrc vendored Normal file
View File

@@ -0,0 +1,2 @@
save-exact=true
min-release-age=2

351
third_party/pi/.pi/extensions/import-repro.ts vendored Normal file
View File

@@ -0,0 +1,351 @@
/**
* Import a pi session shared as a gist by the issue-analysis CI workflow
* (.github/workflows/issue-analysis.yml) and switch to it.
*
* The CI job runs in a high-entropy checkout directory; this command rewrites
* the recorded cwd to the local checkout, installs the session file into the
* current session directory, and switches to it.
*
* Usage:
* /ir b4d100022aefb12f25dd2d8485e0a82a
* /ir https://gist.github.com/mitsuhiko/b4d100022aefb12f25dd2d8485e0a82a
* /ir https://pi.dev/session/#b4d100022aefb12f25dd2d8485e0a82a
* /ir https://github.com/earendil-works/pi/issues/123
*
* pi "/ir <gist-id>"
*/
import { Buffer } from "node:buffer";
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { basename, isAbsolute, join, resolve } from "node:path";
import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
const GIST_ID_RE = /^[0-9a-fA-F]{20,}$/;
const GIST_URL_RE = /^https:\/\/gist\.github\.com\/(?:[^/]+\/)?([0-9a-fA-F]{20,})(?:[/#?].*)?$/;
const SHARE_URL_RE = /^https:\/\/pi\.dev\/session\/#([0-9a-fA-F]{20,})(?:[/#?].*)?$/;
const ISSUE_URL_RE = /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/issues\/(\d+)(?:[/#?].*)?$/;
const GIST_URL_IN_TEXT_RE = /https:\/\/gist\.github\.com\/(?:[^/\s]+\/)?([0-9a-fA-F]{20,})\b/g;
const SESSION_DATA_RE = /<script id="session-data" type="application\/json">([^<]+)<\/script>/;
interface SessionHeader {
type: "session";
id: string;
cwd: string;
[key: string]: unknown;
}
interface ExportedSessionData {
header: SessionHeader | null;
entries: Array<Record<string, unknown>>;
}
interface GistFile {
filename?: string;
raw_url?: string;
content?: string;
truncated?: boolean;
}
interface GistResponse {
files?: Record<string, GistFile>;
}
interface IssueComment {
body?: string | null;
user?: { login?: string } | null;
}
function parseRef(
ref: string,
cwd: string,
): { type: "gist"; id: string } | { type: "file"; path: string } | { type: "issue"; owner: string; repo: string; issue: string } {
if (ref.endsWith(".html") || ref.endsWith(".jsonl")) {
return { type: "file", path: isAbsolute(ref) ? ref : resolve(cwd, ref) };
}
const shareMatch = ref.match(SHARE_URL_RE);
if (shareMatch) return { type: "gist", id: shareMatch[1] };
const gistMatch = ref.match(GIST_URL_RE);
if (gistMatch) return { type: "gist", id: gistMatch[1] };
const issueMatch = ref.match(ISSUE_URL_RE);
if (issueMatch) return { type: "issue", owner: issueMatch[1], repo: issueMatch[2], issue: issueMatch[3] };
if (GIST_ID_RE.test(ref)) return { type: "gist", id: ref };
throw new Error(`expected a gist ID, gist URL, pi.dev share URL, issue URL, .html file, or .jsonl file: ${ref}`);
}
function parseSessionJsonl(raw: string): { header: SessionHeader; jsonl: string } {
const newlineIndex = raw.indexOf("\n");
const firstLine = newlineIndex === -1 ? raw : raw.slice(0, newlineIndex);
let parsed: unknown;
try {
parsed = JSON.parse(firstLine);
} catch {
throw new Error("first line of session file is not valid JSON");
}
const header = parsed as Partial<SessionHeader>;
if (header.type !== "session" || typeof header.id !== "string" || typeof header.cwd !== "string" || header.cwd === "") {
throw new Error("session file has no valid session header with a cwd");
}
return { header: header as SessionHeader, jsonl: raw };
}
function decodeExportedHtml(html: string): { header: SessionHeader; jsonl: string } {
const match = html.match(SESSION_DATA_RE);
if (!match) throw new Error("HTML does not contain embedded pi session data");
let data: unknown;
try {
data = JSON.parse(Buffer.from(match[1], "base64").toString("utf8"));
} catch {
throw new Error("embedded pi session data is not valid JSON");
}
const sessionData = data as Partial<ExportedSessionData>;
const header = sessionData.header;
if (!header || header.type !== "session" || typeof header.id !== "string" || typeof header.cwd !== "string") {
throw new Error("embedded pi session data has no valid session header");
}
if (!Array.isArray(sessionData.entries)) {
throw new Error("embedded pi session data has no entries array");
}
const lines = [header, ...sessionData.entries].map((entry) => JSON.stringify(entry));
return { header, jsonl: `${lines.join("\n")}\n` };
}
type SessionPlatform = "windows" | "unix" | "unknown";
function escapeJsonString(value: string): string {
return JSON.stringify(value).slice(1, -1);
}
function escapeRegExp(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function trimTrailingPathSeparators(value: string): string {
return value.replace(/[\\/]+$/, "");
}
function getPathTailName(value: string): string {
const trimmed = trimTrailingPathSeparators(value);
return trimmed.split(/[\\/]/).filter(Boolean).at(-1) ?? "";
}
function getWindowsDrivePathParts(value: string): { drive: string; rest: string } | undefined {
const trimmed = trimTrailingPathSeparators(value);
const driveMatch = trimmed.match(/^([A-Za-z]):[\\/](.*)$/);
if (driveMatch) {
return { drive: driveMatch[1].toUpperCase(), rest: driveMatch[2].replace(/[\\/]+/g, "/") };
}
const msysMatch = trimmed.match(/^\/([A-Za-z])\/(.*)$/);
if (msysMatch) {
return { drive: msysMatch[1].toUpperCase(), rest: msysMatch[2].replace(/[\\/]+/g, "/") };
}
return undefined;
}
function getCwdRewriteVariants(sourceCwd: string): string[] {
const trimmed = trimTrailingPathSeparators(sourceCwd);
const variants = new Set<string>();
if (trimmed) variants.add(trimmed);
const driveParts = getWindowsDrivePathParts(trimmed);
if (driveParts) {
const rest = driveParts.rest.replace(/^\/+|\/+$/g, "");
const backslashRest = rest.replace(/\//g, "\\");
variants.add(`${driveParts.drive}:\\${backslashRest}`);
variants.add(`${driveParts.drive}:/${rest}`);
variants.add(`/${driveParts.drive.toLowerCase()}/${rest}`);
variants.add(`/${driveParts.drive}/${rest}`);
}
return Array.from(variants).filter(Boolean).sort((a, b) => b.length - a.length);
}
function getCiWorkdirName(sourceCwd: string): string | undefined {
const name = getPathTailName(sourceCwd);
return /^pi-ci-[0-9a-f]{32}$/i.test(name) ? name : undefined;
}
function detectSessionPlatform(cwd: string): SessionPlatform {
if (/^[A-Za-z]:[\\/]/.test(cwd) || /^\/[A-Za-z]\//.test(cwd)) return "windows";
if (cwd.startsWith("/")) return "unix";
return "unknown";
}
function getLocalPlatform(): Exclude<SessionPlatform, "unknown"> {
return process.platform === "win32" ? "windows" : "unix";
}
function getPlatformContinuationNotice(sourceCwd: string): string | undefined {
const sourcePlatform = detectSessionPlatform(sourceCwd);
const localPlatform = getLocalPlatform();
if (sourcePlatform === "unknown" || sourcePlatform === localPlatform) return undefined;
if (localPlatform === "unix") {
return "This session was continued on a non-Windows machine; paths are now Unix style.";
}
return "This session was continued on a Windows machine; paths are now Windows style.";
}
/** Rewrite occurrences of the recorded CI cwd (JSON-escaped) to the target cwd. */
function rewriteSessionCwd(raw: string, sourceCwd: string, targetCwd: string): string {
const target = escapeJsonString(targetCwd);
let rewritten = raw;
for (const sourceVariant of getCwdRewriteVariants(sourceCwd)) {
if (sourceVariant === targetCwd) continue;
rewritten = rewritten.split(escapeJsonString(sourceVariant)).join(target);
}
const ciWorkdirName = getCiWorkdirName(sourceCwd);
if (ciWorkdirName) {
const escapedName = escapeRegExp(ciWorkdirName);
const windowsPathPatterns = [
new RegExp(`[A-Za-z]:(?:[^"\\r\\n])*?${escapedName}`, "g"),
new RegExp(`/[A-Za-z]/(?:[^"\\r\\n])*?${escapedName}`, "g"),
];
for (const pattern of windowsPathPatterns) {
rewritten = rewritten.replace(pattern, target);
}
}
return rewritten;
}
async function fetchText(url: string): Promise<string> {
const response = await fetch(url, { headers: { Accept: "application/vnd.github+json" } });
if (!response.ok) {
throw new Error(`failed to fetch ${url}: HTTP ${response.status}`);
}
return await response.text();
}
async function readGistFile(file: GistFile): Promise<string> {
if (file.content && !file.truncated) return file.content;
if (!file.raw_url) throw new Error(`gist file ${file.filename ?? "<unknown>"} has no raw URL`);
return await fetchText(file.raw_url);
}
async function findIssueGistId(owner: string, repo: string, issue: string): Promise<string> {
const gistIds: string[] = [];
let page = 1;
while (true) {
const response = await fetch(
`https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues/${encodeURIComponent(issue)}/comments?per_page=100&page=${page}`,
{ headers: { Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" } },
);
if (!response.ok) throw new Error(`failed to fetch issue comments: HTTP ${response.status}`);
const comments = (await response.json()) as IssueComment[];
for (const comment of comments) {
if (comment.user?.login !== "github-actions[bot]") continue;
for (const match of (comment.body ?? "").matchAll(GIST_URL_IN_TEXT_RE)) {
gistIds.push(match[1]);
}
}
if (comments.length < 100) break;
page++;
}
const gistId = gistIds.at(-1);
if (!gistId) throw new Error(`no github-actions gist link found in comments on ${owner}/${repo}#${issue}`);
return gistId;
}
async function fetchGistSession(gistId: string): Promise<{ header: SessionHeader; jsonl: string }> {
const response = await fetch(`https://api.github.com/gists/${gistId}`, {
headers: {
Accept: "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
},
});
if (!response.ok) throw new Error(`failed to fetch gist ${gistId}: HTTP ${response.status}`);
const gist = (await response.json()) as GistResponse;
const files = Object.values(gist.files ?? {});
const jsonlFile = files.find((file) => file.filename?.endsWith(".jsonl"));
if (jsonlFile) return parseSessionJsonl(await readGistFile(jsonlFile));
const htmlFile = files.find((file) => file.filename?.endsWith(".html"));
if (htmlFile) return decodeExportedHtml(await readGistFile(htmlFile));
throw new Error(`gist ${gistId} has no .jsonl or .html session file`);
}
export default function (pi: ExtensionAPI) {
pi.registerCommand("ir", {
description: "Import a CI issue-analysis session from a gist ID, share URL, or issue URL and switch to it",
handler: async (args: string, ctx: ExtensionCommandContext) => {
const ref = args.trim();
if (!ref) {
ctx.ui.notify("Usage: /ir <gist-id | gist-url | pi.dev/session URL | issue URL>", "error");
return;
}
try {
const targetCwd = ctx.sessionManager.getCwd();
const sessionDir = ctx.sessionManager.getSessionDir();
const parsedRef = parseRef(ref, targetCwd);
ctx.ui.notify(`Importing repro session from ${ref}...`, "info");
let sourceName: string;
let decoded: { header: SessionHeader; jsonl: string };
if (parsedRef.type === "gist") {
decoded = await fetchGistSession(parsedRef.id);
sourceName = `${parsedRef.id}.jsonl`;
} else if (parsedRef.type === "issue") {
const gistId = await findIssueGistId(parsedRef.owner, parsedRef.repo, parsedRef.issue);
decoded = await fetchGistSession(gistId);
sourceName = `${gistId}.jsonl`;
} else {
if (!existsSync(parsedRef.path)) throw new Error(`session file not found: ${parsedRef.path}`);
const raw = readFileSync(parsedRef.path, "utf8");
decoded = parsedRef.path.endsWith(".html") ? decodeExportedHtml(raw) : parseSessionJsonl(raw);
sourceName = basename(parsedRef.path).replace(/\.html$/, ".jsonl");
}
const platformNotice = getPlatformContinuationNotice(decoded.header.cwd);
const rewritten = rewriteSessionCwd(decoded.jsonl, decoded.header.cwd, targetCwd);
const destination = join(sessionDir, sourceName);
if (existsSync(destination)) {
const overwrite = await ctx.ui.confirm(
"Session already imported",
`Overwrite ${destination}? Local changes to that session will be lost.`,
);
if (!overwrite) {
ctx.ui.notify("Import cancelled", "warning");
return;
}
}
writeFileSync(destination, rewritten);
ctx.ui.notify(`Imported session ${decoded.header.id} (cwd ${decoded.header.cwd} -> ${targetCwd})`, "info");
await ctx.switchSession(destination, {
withSession: async (nextCtx) => {
if (!platformNotice) return;
await nextCtx.sendMessage(
{
customType: "import-repro",
content: platformNotice,
display: true,
details: { sourceCwd: decoded.header.cwd, targetCwd },
},
{ triggerTurn: false },
);
},
});
} catch (error) {
ctx.ui.notify(`ir: ${error instanceof Error ? error.message : String(error)}`, "error");
}
},
});
}

270
third_party/pi/.pi/extensions/prompt-url-widget.ts vendored Normal file
View File

@@ -0,0 +1,270 @@
import { readFile } from "node:fs/promises";
import { homedir } from "node:os";
import { resolve } from "node:path";
import { DynamicBorder, type ExtensionAPI, type ExtensionContext } from "@earendil-works/pi-coding-agent";
import { Container, hyperlink, Text } from "@earendil-works/pi-tui";
const PR_PROMPT_PATTERN = /^\s*You are given one or more GitHub PR URLs:\s*(\S+)/im;
const ISSUE_PROMPT_PATTERN = /^\s*Analyze GitHub issue\(s\):\s*(\S+)/im;
const ADVISORY_PROMPT_PATTERN = /^\s*Update a GitHub security advisory for publication:\s*(\S+)/im;
type PromptMatch = {
kind: "pr" | "issue" | "advisory";
target: string;
};
type GhMetadata = {
title?: string;
detail?: string;
displayUrl?: string;
author?: {
login?: string;
name?: string | null;
};
};
type GitHubAdvisoryMetadata = {
ghsa_id?: string;
summary?: string;
severity?: string;
state?: string;
html_url?: string;
cve_id?: string | null;
};
type AdvisoryRef = {
owner: string;
repo: string;
ghsaId: string;
url: string;
};
function extractPromptMatch(prompt: string): PromptMatch | undefined {
const prMatch = prompt.match(PR_PROMPT_PATTERN);
if (prMatch?.[1]) {
return { kind: "pr", target: prMatch[1].trim() };
}
const issueMatch = prompt.match(ISSUE_PROMPT_PATTERN);
if (issueMatch?.[1]) {
return { kind: "issue", target: issueMatch[1].trim() };
}
const advisoryMatch = prompt.match(ADVISORY_PROMPT_PATTERN);
if (advisoryMatch?.[1]) {
return { kind: "advisory", target: advisoryMatch[1].trim() };
}
return undefined;
}
function getPromptLabel(kind: PromptMatch["kind"]): string {
if (kind === "pr") return "PR";
if (kind === "issue") return "Issue";
return "Advisory";
}
function parseAdvisoryUrl(value: string): AdvisoryRef | undefined {
const match = value.match(
/^https:\/\/github\.com\/([^/]+)\/([^/]+)\/security\/advisories\/(GHSA-[A-Za-z0-9-]+)(?:[/?#].*)?$/i,
);
if (!match?.[1] || !match[2] || !match[3]) return undefined;
return {
owner: match[1],
repo: match[2],
ghsaId: match[3],
url: `https://github.com/${match[1]}/${match[2]}/security/advisories/${match[3]}`,
};
}
function unquoteYamlValue(value: string): string {
const trimmed = value.trim();
if (
(trimmed.startsWith('"') && trimmed.endsWith('"')) ||
(trimmed.startsWith("'") && trimmed.endsWith("'"))
) {
return trimmed.slice(1, -1);
}
return trimmed;
}
function resolveDraftPath(cwd: string, target: string): string {
if (target === "~") return homedir();
if (target.startsWith("~/")) return resolve(homedir(), target.slice(2));
return resolve(cwd, target);
}
async function readAdvisoryRefFromDraft(cwd: string, target: string): Promise<AdvisoryRef | undefined> {
try {
const content = await readFile(resolveDraftPath(cwd, target), "utf8");
const frontmatter = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
const body = frontmatter?.[1] ?? content;
const urlMatch = body.match(/^advisory_url:\s*(.+)$/m);
if (!urlMatch?.[1]) return undefined;
return parseAdvisoryUrl(unquoteYamlValue(urlMatch[1]));
} catch {
return undefined;
}
}
function formatAdvisoryDetail(advisory: GitHubAdvisoryMetadata): string | undefined {
const parts = [advisory.ghsa_id, advisory.cve_id ?? undefined, advisory.severity, advisory.state]
.map((part) => part?.trim())
.filter((part): part is string => part !== undefined && part.length > 0);
return parts.length > 0 ? parts.join(" · ") : undefined;
}
async function fetchAdvisoryMetadata(pi: ExtensionAPI, cwd: string, target: string): Promise<GhMetadata | undefined> {
const advisoryRef = parseAdvisoryUrl(target) ?? (await readAdvisoryRefFromDraft(cwd, target));
if (!advisoryRef) return undefined;
try {
const result = await pi.exec("gh", [
"api",
`repos/${advisoryRef.owner}/${advisoryRef.repo}/security-advisories/${advisoryRef.ghsaId}`,
]);
if (result.code !== 0 || !result.stdout) return { displayUrl: advisoryRef.url };
const advisory = JSON.parse(result.stdout) as GitHubAdvisoryMetadata;
return {
title: advisory.summary,
detail: formatAdvisoryDetail(advisory),
displayUrl: advisory.html_url ?? advisoryRef.url,
};
} catch {
return { displayUrl: advisoryRef.url };
}
}
async function fetchGhMetadata(
pi: ExtensionAPI,
kind: PromptMatch["kind"],
target: string,
cwd: string,
): Promise<GhMetadata | undefined> {
if (kind === "advisory") {
return fetchAdvisoryMetadata(pi, cwd, target);
}
const args =
kind === "pr"
? ["pr", "view", target, "--json", "title,author"]
: ["issue", "view", target, "--json", "title,author"];
try {
const result = await pi.exec("gh", args);
if (result.code !== 0 || !result.stdout) return undefined;
return JSON.parse(result.stdout) as GhMetadata;
} catch {
return undefined;
}
}
function formatAuthor(author?: GhMetadata["author"]): string | undefined {
if (!author) return undefined;
const name = author.name?.trim();
const login = author.login?.trim();
if (name && login) return `${name} (@${login})`;
if (login) return `@${login}`;
if (name) return name;
return undefined;
}
export default function promptUrlWidgetExtension(pi: ExtensionAPI) {
const setWidget = (ctx: ExtensionContext, match: PromptMatch, metadata?: GhMetadata) => {
ctx.ui.setWidget("prompt-url", (_tui, thm) => {
const displayTarget = metadata?.displayUrl ?? match.target;
const titleText = metadata?.title
? thm.fg("accent", metadata.title)
: hyperlink(thm.fg("accent", displayTarget), displayTarget);
const detailText = metadata?.detail ?? formatAuthor(metadata?.author);
const detailLine = detailText ? thm.fg("muted", detailText) : undefined;
const urlLine = hyperlink(thm.fg("dim", displayTarget), displayTarget);
const lines = [titleText];
if (detailLine) lines.push(detailLine);
lines.push(urlLine);
const container = new Container();
container.addChild(new DynamicBorder((s: string) => thm.fg("muted", s)));
container.addChild(new Text(lines.join("\n"), 1, 0));
return container;
});
};
const applySessionName = (ctx: ExtensionContext, match: PromptMatch, metadata?: GhMetadata) => {
const label = getPromptLabel(match.kind);
const displayTarget = metadata?.displayUrl ?? match.target;
const trimmedTitle = metadata?.title?.trim();
const fallbackName = `${label}: ${match.target}`;
const desiredFallbackName = `${label}: ${displayTarget}`;
const desiredName = trimmedTitle ? `${label}: ${trimmedTitle} (${displayTarget})` : desiredFallbackName;
const currentName = pi.getSessionName()?.trim();
if (!currentName) {
pi.setSessionName(desiredName);
return;
}
if (currentName === match.target || currentName === fallbackName || currentName === desiredFallbackName) {
pi.setSessionName(desiredName);
}
};
const updatePromptContext = (ctx: ExtensionContext, match: PromptMatch) => {
setWidget(ctx, match);
applySessionName(ctx, match);
void fetchGhMetadata(pi, match.kind, match.target, ctx.cwd).then((meta) => {
setWidget(ctx, match, meta);
applySessionName(ctx, match, meta);
});
};
pi.on("before_agent_start", async (event, ctx) => {
if (!ctx.hasUI) return;
const match = extractPromptMatch(event.prompt);
if (!match) {
return;
}
updatePromptContext(ctx, match);
});
pi.on("session_switch", async (_event, ctx) => {
rebuildFromSession(ctx);
});
const getUserText = (content: string | { type: string; text?: string }[] | undefined): string => {
if (!content) return "";
if (typeof content === "string") return content;
return (
content
.filter((block): block is { type: "text"; text: string } => block.type === "text")
.map((block) => block.text)
.join("\n") ?? ""
);
};
const rebuildFromSession = (ctx: ExtensionContext) => {
if (!ctx.hasUI) return;
const entries = ctx.sessionManager.getEntries();
const lastMatch = [...entries].reverse().find((entry) => {
if (entry.type !== "message" || entry.message.role !== "user") return false;
const text = getUserText(entry.message.content);
return !!extractPromptMatch(text);
});
const content =
lastMatch?.type === "message" && lastMatch.message.role === "user" ? lastMatch.message.content : undefined;
const text = getUserText(content);
const match = text ? extractPromptMatch(text) : undefined;
if (!match) {
ctx.ui.setWidget("prompt-url", undefined);
return;
}
updatePromptContext(ctx, match);
};
pi.on("session_start", async (_event, ctx) => {
rebuildFromSession(ctx);
});
}

24
third_party/pi/.pi/extensions/redraws.ts vendored Normal file
View File

@@ -0,0 +1,24 @@
/**
* Redraws Extension
*
* Exposes /tui to show TUI redraw stats.
*/
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
import { Text } from "@earendil-works/pi-tui";
export default function (pi: ExtensionAPI) {
pi.registerCommand("tui", {
description: "Show TUI stats",
handler: async (_args, ctx) => {
if (!ctx.hasUI) return;
let redraws = 0;
await ctx.ui.custom<void>((tui, _theme, _keybindings, done) => {
redraws = tui.fullRedraws;
done(undefined);
return new Text("", 0, 0);
});
ctx.ui.notify(`TUI full redraws: ${redraws}`, "info");
},
});
}

47
third_party/pi/.pi/extensions/tps.ts vendored Normal file
View File

@@ -0,0 +1,47 @@
import type { AssistantMessage } from "@earendil-works/pi-ai";
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
function isAssistantMessage(message: unknown): message is AssistantMessage {
if (!message || typeof message !== "object") return false;
const role = (message as { role?: unknown }).role;
return role === "assistant";
}
export default function (pi: ExtensionAPI) {
let agentStartMs: number | null = null;
pi.on("agent_start", () => {
agentStartMs = Date.now();
});
pi.on("agent_end", (event, ctx) => {
if (!ctx.hasUI) return;
if (agentStartMs === null) return;
const elapsedMs = Date.now() - agentStartMs;
agentStartMs = null;
if (elapsedMs <= 0) return;
let input = 0;
let output = 0;
let cacheRead = 0;
let cacheWrite = 0;
let totalTokens = 0;
for (const message of event.messages) {
if (!isAssistantMessage(message)) continue;
input += message.usage.input || 0;
output += message.usage.output || 0;
cacheRead += message.usage.cacheRead || 0;
cacheWrite += message.usage.cacheWrite || 0;
totalTokens += message.usage.totalTokens || 0;
}
if (output <= 0) return;
const elapsedSeconds = elapsedMs / 1000;
const tokensPerSecond = output / elapsedSeconds;
const message = `TPS ${tokensPerSecond.toFixed(1)} tok/s. out ${output.toLocaleString()}, in ${input.toLocaleString()}, cache r/w ${cacheRead.toLocaleString()}/${cacheWrite.toLocaleString()}, total ${totalTokens.toLocaleString()}, ${elapsedSeconds.toFixed(1)}s`;
ctx.ui.notify(message, "info");
});
}

2
third_party/pi/.pi/git/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
*
!.gitignore

2
third_party/pi/.pi/npm/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
*
!.gitignore

54
third_party/pi/.pi/prompts/cl.md vendored Normal file
View File

@@ -0,0 +1,54 @@
---
description: Audit changelog entries before release
---
Audit changelog entries for all commits since the last release.
## Process
1. **Find the last release tag:**
```bash
git tag --sort=-version:refname | head -1
```
2. **List all commits since that tag:**
```bash
git log <tag>..HEAD --oneline
```
3. **Read each package's [Unreleased] section:**
- packages/ai/CHANGELOG.md
- packages/tui/CHANGELOG.md
- packages/coding-agent/CHANGELOG.md
4. **For each commit, check:**
- Skip: changelog updates, doc-only changes, release housekeeping
- Skip: changes to generated model catalogs (for example `packages/ai/src/models.generated.ts`) unless accompanied by an intentional product-facing change in non-generated source/docs.
- Determine which package(s) the commit affects (use `git show <hash> --stat`)
- Verify a changelog entry exists in the affected package(s)
- For external contributions (PRs), verify format: `Description ([#N](url) by [@user](url))`
5. **Cross-package duplication rule:**
Changes in `ai`, `agent` or `tui` that affect end users should be duplicated to `coding-agent` changelog, since coding-agent is the user-facing package that depends on them.
6. **Add New Features section after changelog fixes:**
- Insert a `### New Features` section at the start of `## [Unreleased]` in `packages/coding-agent/CHANGELOG.md`.
- Propose the top new features to the user for confirmation before writing them.
- Link to relevant docs and sections whenever possible.
7. **Report:**
- List commits with missing entries
- List entries that need cross-package duplication
- Add any missing entries directly
## Changelog Format Reference
Sections (in order):
- `### Breaking Changes` - API changes requiring migration
- `### Added` - New features
- `### Changed` - Changes to existing functionality
- `### Fixed` - Bug fixes
- `### Removed` - Removed features
Attribution:
- Internal: `Fixed foo ([#123](https://github.com/earendil-works/pi-mono/issues/123))`
- External: `Added bar ([#456](https://github.com/earendil-works/pi-mono/pull/456) by [@user](https://github.com/user))`

67
third_party/pi/.pi/prompts/deslop.md vendored Normal file
View File

@@ -0,0 +1,67 @@
---
description: simplify a completed workpackage
---
Review the completed workpackage and simplify its implementation without changing its intended behavior.
## Scope
- Focus on code added or affected by the workpackage and the nearby code needed to simplify it coherently.
- Do not perform unrelated cleanup.
- Read the affected files in full and understand the relevant invariants, call sites, tests, and public API before editing.
- Optimize for a direct, coherent, maintainable design—not the smallest diff.
## What to remove or simplify
- Remove abstractions that have no clear responsibility or do not reduce real complexity.
- Inline trivial helpers, wrappers, adapters, and pass-through layers when they obscure rather than clarify behavior.
- Remove duplicated logic and put shared behavior at the appropriate existing layer.
- Remove speculative flexibility, configuration, extension points, and generic machinery that have no current requirement.
- Remove speculative defensive programming inside trusted code. Validate at real trust boundaries; rely on types and established invariants internally.
- Do not keep fallback behavior for states that should be impossible. Prefer fixing the invariant or type model.
- Simplify excessive state, boolean flags, branching, indirection, and special cases. Rework the underlying representation when that is clearer.
- Tighten TypeScript types so invalid states are unrepresentable. Avoid `any`, unnecessary assertions, broad types, and optional fields for impossible states.
- Prefer direct, readable code over cleverness, premature abstraction, and framework-like infrastructure.
- Delete comments that merely restate the code, but preserve comments that explain constraints, intent, or non-obvious decisions.
## Approval gate for significant removals
Before making any significant or potentially intentional removal, stop and ask the user for approval. First explain:
1. what you propose to remove or change;
2. why it appears unnecessary or overly defensive;
3. what behavior, compatibility, extensibility, validation, or failure handling could be affected;
4. the simpler replacement, if any; and
5. your recommendation.
Wait for explicit approval before applying that change. Do not bundle approval for multiple independent decisions; present them separately when the tradeoffs differ.
Treat a decision as significant when it removes or materially changes any of the following:
- user-visible behavior or supported workflows;
- public APIs, persisted formats, protocol behavior, or backward compatibility;
- validation, authorization, security checks, recovery, retries, fallbacks, or error handling;
- an abstraction or extension point that appears deliberate or has multiple consumers;
- functionality covered by tests or documentation;
- code whose purpose or invariant is uncertain.
Routine, behavior-preserving cleanup does not require approval, such as inlining a single-use trivial helper, removing an unreachable branch proven impossible by the type model, or deleting newly added duplication. If uncertain whether a removal is significant, ask.
## Constraints
- Preserve intended behavior unless the user explicitly approves a change.
- Do not remove code merely because it is unused until you have checked whether it is a public API, extension point, generated entry point, or intentionally retained compatibility surface.
- Do not replace clear code with a new abstraction solely to reduce line count.
- Do not weaken tests to permit simplification. Update tests only when an approved behavior change or a cleaner equivalent structure requires it.
- Follow the repository's validation and testing instructions after edits.
## Completion
After editing, review the final diff for avoidable complexity, duplication, indirection, defensive branches, and accidental behavior changes. Report:
- what was simplified;
- which invariants the implementation now relies on;
- any significant removals you did not make because approval was not granted; and
- the validation performed and its results.
Passing tests and type checks is necessary but not sufficient. Leave the affected code easier to understand and change.

28
third_party/pi/.pi/prompts/is.md vendored Normal file
View File

@@ -0,0 +1,28 @@
---
description: Analyze GitHub issues (bugs or feature requests)
argument-hint: "<issue>"
---
Analyze GitHub issue(s): $ARGUMENTS
For each issue:
1. If running under CI (`CI=true`), do not add the `inprogress` label and do not assign the issue. Otherwise, add the `inprogress` label to the issue via GitHub CLI and assign the issue to the local `gh` user before analysis starts. If either action fails, report that explicitly and continue.
2. Read the issue in full, including all comments and linked issues/PRs. Use fields supported by GitHub CLI, for example:
```sh
gh issue view <issue> --json title,body,comments,labels,assignees,state,url,author,createdAt,updatedAt,closedByPullRequestsReferences
```
3. Do not trust analysis written in the issue. Independently verify behavior and derive your own analysis from the code and execution path.
4. **For bugs**:
- Ignore any root cause analysis in the issue (likely wrong)
- Read all related code files in full (no truncation)
- Trace the code path and identify the actual root cause
- Propose a fix
5. **For feature requests**:
- Do not trust implementation proposals in the issue without verification
- Read all related code files in full (no truncation)
- Propose the most concise implementation approach
- List affected files and changes needed
Do NOT implement unless explicitly asked. Analyze and propose only.

37
third_party/pi/.pi/prompts/pr.md vendored Normal file
View File

@@ -0,0 +1,37 @@
---
description: Review PRs from URLs with structured issue and code analysis
argument-hint: "<PR-URL>"
---
You are given one or more GitHub PR URLs: $@
For each PR URL, do the following in order:
1. Add the `inprogress` label to the PR via GitHub CLI before analysis starts. If adding the label fails, report that explicitly and continue.
2. Read the PR page in full. Include description, all comments, all commits, and all changed files.
3. Identify any linked issues referenced in the PR body, comments, commit messages, or cross links. Read each issue in full, including all comments.
4. Analyze the PR diff without checking out or switching to the PR branch. Use `gh pr diff`, `gh pr view`, `gh api`, and local main-branch files; if PR file contents are needed, use fetched refs with `git show <ref>:<path>` or temporary files. Read all relevant code files in full with no truncation and compare against the diff. Do not fetch PR file blobs unless a file is missing on main or the diff context is insufficient. Include related code paths that are not in the diff but are required to validate behavior.
5. Do not check for a changelog entry. Per CONTRIBUTING.md, contributor PRs must not edit `CHANGELOG.md` — the maintainer adds the entry when merging.
6. Check if packages/coding-agent/README.md, packages/coding-agent/docs/*.md, packages/coding-agent/examples/**/*.md require modification. This is usually the case when existing features have been changed, or new features have been added.
7. Provide a structured review with these sections:
- What it does: one short paragraph describing the change and its intent.
- Good: solid choices or improvements.
- Bad: concrete issues, regressions, missing tests, or risks.
- Ugly: subtle or high impact problems.
- Tests: what is covered, what is missing, and whether existing tests are adequate.
- Open questions for you: only things blocking a merge decision that need the user's input. Omit the section entirely if there are none.
Output format per PR:
PR: <url>
What it does:
- ...
Good:
- ...
Bad:
- ...
Ugly:
- ...
Tests:
- ...
Open questions for you:
- ...
If no issues are found, say so under Bad and Ugly.

163
third_party/pi/.pi/prompts/sa.md vendored Normal file
View File

@@ -0,0 +1,163 @@
---
description: Update a GitHub security advisory for publication
argument-hint: "<advisory-url-or-draft-path>"
---
Update a GitHub security advisory for publication: $ARGUMENTS
Use `gh` for all GitHub operations. Do not publish the advisory, change its state, or request a CVE unless the user explicitly agrees or the draft markdown explicitly says `request_cve: true`.
GitHub does not expose repository security advisory comments/discussion through the documented REST OpenAPI schema or public GraphQL schema. A 404 from guessed API endpoints such as `api.github.com/repos/.../security-advisories/<GHSA>/comments`, `.../timeline`, or `.../events` is expected and is not, by itself, an auth failure. Do not use a browser session, browser cookies, or cookie extraction to fetch advisory comments. Instead, clearly tell the user that advisory comments were not included and that they can paste any relevant comments if they want them considered.
## Input handling
- If `$ARGUMENTS` is a GitHub security advisory URL, start the investigation and drafting workflow.
- If `$ARGUMENTS` is a path to an existing markdown draft, read it and apply that draft to the advisory.
- In a follow-up message after this prompt, if the user says "update", "apply", "looks good", or similar, treat it as approval to apply the previously written temp markdown draft. Re-read the file from disk before updating GitHub.
- If applying a draft and there is no known draft path, ask the user for the markdown file path.
## Initial advisory workflow
1. Parse the advisory URL into `owner`, `repo`, and `GHSA` id.
2. Fetch the advisory with:
```sh
gh api repos/<owner>/<repo>/security-advisories/<GHSA>
```
Record the advisory's original severity, CVSS vector, and CVSS score exactly as returned before proposing changes.
3. Do not fetch advisory comments/discussion unless the user pasted them into the conversation:
- Inspect the advisory JSON for references, credits, linked issues/PRs, and any discussion fields.
- Do not rely on invented API endpoints such as `/comments`, `/timeline`, or `/events`; they commonly return 404 because GitHub does not expose draft advisory comments through the public API.
- Do not use a browser session, browser cookies, or cookie extraction to fetch comments.
- Explicitly tell the user: `Advisory comments were not included because GitHub does not expose them through the public API. Paste any relevant comments if you want them considered.`
- If the user pasted comments, read and consider them.
- Never pretend comments were read.
4. Investigate independently:
- Read the advisory text, metadata, affected package(s), version ranges, CVSS, CWE, references, and linked issues/PRs/commits.
- Inspect relevant code history, releases, changelogs, package metadata, and tags.
- Determine whether the vulnerability is already fixed.
- If fixed, identify the patched version(s) and the correct affected version range.
- Do not trust the reporter's analysis without verification.
5. Discuss CVSS with the user before drafting the final update:
- Propose a CVSS vector, score, and severity.
- Explain the controversial metrics briefly.
- Ask the user to confirm or adjust it.
6. Ask whether a CVE should be requested from GitHub for this advisory.
7. Draft a publication-ready advisory markdown file under `/tmp`, for example `/tmp/sa-<GHSA>.md`. Include both the original CVSS from the advisory and the proposed/confirmed updated CVSS.
8. Tell the user:
- the path to the temp markdown file
- the original advisory URL
- that they can edit the file and then say "update" or provide the path
## Draft markdown format
The draft file must contain YAML frontmatter followed by the advisory body. Include all fields needed to update GitHub and to decide whether to request a CVE.
```markdown
---
advisory_url: https://github.com/<owner>/<repo>/security/advisories/<GHSA>
owner: <owner>
repo: <repo>
ghsa_id: <GHSA>
summary: <short advisory summary>
original_severity: <low|medium|high|critical|null>
original_cvss_vector: <original CVSS:3.1/... or null>
original_cvss_score: <original number or null>
severity: <proposed/confirmed low|medium|high|critical>
cvss_vector: <proposed/confirmed CVSS:3.1/...>
cvss_score: <proposed/confirmed number>
cwe_ids:
- CWE-...
vulnerabilities:
- package:
ecosystem: npm
name: <package-name>
vulnerable_version_range: <range>
patched_versions: <range-or-version>
request_cve: false
---
# <Advisory title>
<Concise description of the vulnerability and vulnerable behavior.>
## Info
<Technical explanation of the root cause and affected component. Focus on facts needed by defenders and maintainers. Do not include PoC steps, exploit payloads, or copy-pastable exploit strings.>
## Impact
<Who can exploit it, prerequisites, confidentiality/integrity/availability impact, and realistic deployment assumptions.>
## Affected versions
- Affected: `<range>`
- Patched: `<version or range>`
## The solution
<Describe the fix and the patched release.>
## Recommendations
<Upgrade guidance and operational mitigations.>
## Workarounds
<Workarounds if any; otherwise skip this section entirely>
## Timeline
- YYYY-MM-DD: Report received
- YYYY-MM-DD: Fix committed
- YYYY-MM-DD: Fixed version released
- YYYY-MM-DD: Advisory published
## Credits
<Reporter/researcher attribution if appropriate, otherwise skip section.>
## References
- <links to releases, commits, advisories, documentation>
```
Use the curl advisory style as inspiration: clear sections, direct language, affected/fixed version facts, recommendations, timeline, and credits. Do not include a PoC.
## Applying a draft to GitHub
When the user approves with "update"/similar or provides a markdown path:
1. Re-read the markdown file from disk. Never rely on the previously generated content in memory.
2. Parse the YAML frontmatter and body.
3. Build a JSON payload in a temporary file. Map fields as follows:
- `summary` from frontmatter
- `description` from the markdown body after frontmatter
- `severity` from frontmatter if present
- `cvss_vector_string` from `cvss_vector`
- `cwe_ids` from frontmatter
- `vulnerabilities` from frontmatter
- Do not send `original_severity`, `original_cvss_vector`, or `original_cvss_score`; those fields are retained only for audit context.
4. Update the advisory with:
```sh
gh api -X PATCH repos/<owner>/<repo>/security-advisories/<GHSA> --input /tmp/<payload>.json
```
5. If and only if the markdown frontmatter has `request_cve: true`, request a CVE with:
```sh
gh api -X POST repos/<owner>/<repo>/security-advisories/<GHSA>/cve
```
Treat "already requested" or "already assigned" as non-fatal and report it.
6. Report what was updated:
- advisory URL
- summary
- affected range
- patched versions
- original CVSS vector/score/severity
- updated CVSS vector/score/severity
- whether CVE was requested
## Safety rules
- Do not include PoC material in the final advisory body.
- Do not request a CVE unless `request_cve: true` is present in the markdown file.
- Do not publish the advisory or change its state unless the user explicitly asks.
- Do not fetch advisory comments through browser sessions or cookies. State that comments were not included and invite the user to paste relevant comments if they want them considered.
- If there is uncertainty in affected ranges, patched versions, CVSS, or CVE request status, ask the user before applying.

41
third_party/pi/.pi/prompts/wr.md vendored Normal file
View File

@@ -0,0 +1,41 @@
---
description: Finish the current task end-to-end with changelog, commit, and push
argument-hint: "[instructions]"
---
Wrap it.
Additional instructions: $ARGUMENTS
Determine context from the conversation history first.
Rules for context detection:
- If the conversation already mentions a GitHub issue or PR, use that existing context.
- If the work came from `/is` or `/pr`, assume the issue or PR context is already known from the conversation and from the analysis work already done.
- If there is no GitHub issue or PR in the conversation history, treat this as non-GitHub work.
Unless I explicitly override something in this request, do the following in order:
1. Add or update the relevant package changelog entry under `## [Unreleased]` using the repo changelog rules.
2. If this task is tied to a GitHub issue or PR and a final issue or PR comment has not already been posted in this session, draft it in my tone, preview it, and post exactly one final comment. The comment must end with this exact standalone disclaimer line, with no variations:
```text
This comment is AI-generated by `/wr`
```
3. Commit only files you changed in this session.
4. If this task is tied to exactly one GitHub issue, include `closes #<issue>` in the commit message. If it is tied to multiple issues, stop and ask which one to use. If it is not tied to any issue, do not include `closes #` or `fixes #` in the commit message.
5. Check the current git branch. If it is not `main`, stop and ask what to do. Do not push from another branch unless I explicitly say so.
6. Push the current branch.
7. If this task is tied to exactly one GitHub issue, explicitly close that issue with reason `completed` after the push so the issue-close workflows in `.github/` run. This applies to issues only, not PRs.
- Inspect `gh issue view <issue> --json state,stateReason,labels`.
- If the issue is open, run `gh issue close <issue> --reason completed`.
- If the issue is already closed with any reason other than `COMPLETED`, reopen it first, then close it with `gh issue close <issue> --reason completed` so GitHub emits a fresh close event.
- If the issue is already closed as `COMPLETED`, leave it closed unless the `inprogress` label is still present; in that case reopen it and close it again with reason `completed`.
Constraints:
- Never stage unrelated files.
- Never use `git add .` or `git add -A`.
- Run required checks before committing if code changed.
- Do not open a PR unless I explicitly ask.
- If this is not GitHub issue or PR work, do not post a GitHub comment.
- If a final issue or PR comment was already posted in this session, do not post another one unless I explicitly ask.
- When working against a branch other than `main`, skip the changelog.

57
third_party/pi/.pi/skills/add-llm-provider.md vendored Normal file
View File

@@ -0,0 +1,57 @@
---
name: add-llm-provider
description: Checklist for adding a new LLM provider to packages/ai. Covers core types, provider implementation, lazy registration, model generation, the full test matrix, coding-agent wiring, and docs.
---
# Adding a New LLM Provider (packages/ai)
A new provider touches multiple files. Work through these steps in order.
## 1. Core Types (`packages/ai/src/types.ts`)
- Add API identifier to `Api` type union (e.g. `"bedrock-converse-stream"`).
- Create options interface extending `StreamOptions`.
- Add mapping to `ApiOptionsMap`.
- Add provider name to `KnownProvider` type union.
## 2. Provider Implementation (`packages/ai/src/providers/`)
Create a provider file exporting:
- `stream<Provider>()` returning `AssistantMessageEventStream`.
- `streamSimple<Provider>()` for `SimpleStreamOptions` mapping.
- Provider-specific options interface.
- Message/tool conversion functions.
- Response parsing that emits standardized events (`text`, `tool_call`, `thinking`, `usage`, `stop`).
## 3. Provider Exports and Lazy Registration
- Add a package subpath export in `packages/ai/package.json` pointing at `./dist/providers/<provider>.js`.
- Add `export type` re-exports in `packages/ai/src/index.ts` for provider option types that should remain available from the root entry.
- Register the provider in `packages/ai/src/providers/register-builtins.ts` via lazy loader wrappers; do not statically import provider implementation modules there.
- Add credential detection in `packages/ai/src/env-api-keys.ts`.
## 4. Model Generation (`packages/ai/scripts/generate-models.ts`)
- Add logic to fetch/parse models from the provider source.
- Map to the standardized `Model` interface.
## 5. Tests (`packages/ai/test/`)
- Always add the provider to `stream.test.ts` with at least one representative model, even if it reuses an existing API impl such as `openai-completions`.
- Add the provider to the broader matrix where applicable: `tokens.test.ts`, `abort.test.ts`, `empty.test.ts`, `context-overflow.test.ts`, `unicode-surrogate.test.ts`, `tool-call-without-result.test.ts`, `image-tool-result.test.ts`, `total-tokens.test.ts`, `cross-provider-handoff.test.ts`.
- For `cross-provider-handoff.test.ts`, add at least one provider/model pair. If the provider exposes multiple model families (e.g. GPT and Claude), add at least one pair per family.
- For non-standard auth, create a utility (e.g. `bedrock-utils.ts`) with credential detection.
## 6. Coding Agent (`packages/coding-agent/`)
- `src/core/model-resolver.ts`: add default model ID to `defaultModelPerProvider`.
- `src/core/provider-display-names.ts`: add API-key login display name so `/login` and related UI show the provider for built-in API-key auth.
- `src/cli/args.ts`: add env var documentation.
- `README.md`: add provider setup instructions.
- `docs/providers.md`: add setup instructions, env var, and `auth.json` key.
## 7. Documentation
- `packages/ai/README.md`: add to providers table, document options/auth, add env vars.
- `packages/ai/CHANGELOG.md`: add entry under `## [Unreleased]`.

169
third_party/pi/AGENTS.md vendored Normal file
View File

@@ -0,0 +1,169 @@
# Development Rules
## Conversational Style
- Keep answers short and concise
- No emojis in commits, issues, PR comments, or code
- No fluff or cheerful filler text (e.g., "Thanks @user" not "Thanks so much @user!")
- Technical prose only, be direct
- Use concise, clear, simple language. Define unavoidable jargon before using it.
- Explain non-trivial designs and problems as: problem, concrete example or short trace, then solution. State why the solution is necessary and distinguish it from optional complexity.
- Prefer concrete behavior and small illustrations over abstract summaries, dense terminology, or unexplained lists of changes.
- When the user asks a question, answer it first before making edits or running implementation commands.
- When responding to user feedback or an analysis, explicitly say whether you agree or disagree before saying what you changed.
## Code Quality
- Read files in full before wide-ranging changes, before editing files you have not fully inspected, and when asked to investigate or audit. Do not rely on search snippets for broad changes.
- No `any` unless absolutely necessary.
- Inline single-line helpers that have only one call site.
- Check node_modules for external API types; don't guess.
- **No inline imports** (`await import()`, `import("pkg").Type`, dynamic type imports). Top-level imports only.
- Never remove or downgrade code to fix type errors from outdated deps; upgrade the dep instead.
- Use only erasable TypeScript syntax (Node strip-only mode) in code checked by the root config (`packages/*/src`, `packages/*/test`, `packages/coding-agent/examples`): no parameter properties, `enum`, `namespace`/`module`, `import =`, `export =`, or other constructs needing JS emit. Use explicit fields with constructor assignments.
- Always ask before removing functionality or code that appears intentional.
- Do not preserve backward compatibility unless the user asks for it.
- Never hardcode key checks (e.g. `matchesKey(keyData, "ctrl+x")`). Add defaults to `DEFAULT_EDITOR_KEYBINDINGS` or `DEFAULT_APP_KEYBINDINGS` so they stay configurable.
- Never modify `packages/ai/src/models.generated.ts` directly; update `packages/ai/scripts/generate-models.ts` instead, then regenerate. Including the resulting `models.generated.ts` diff is always OK, even if regeneration includes unrelated upstream model metadata changes.
## Commands
- After code changes (not docs): `npm run check` (full output, no tail). Fix all errors, warnings, and infos before committing. Does not run tests.
- Never run `npm run build` or `npm test` unless requested by the user.
- Never run the full vitest suite directly: it includes e2e tests that activate when endpoint/auth env vars are present. For all non-e2e tests, run `./test.sh` from the repo root. Otherwise run specific tests from the package root:
- Vitest: `node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/specific.test.ts`
- `packages/tui` (`node:test`): `node --test test/specific.test.ts`
- If you create or modify a test file, run it and iterate on test or implementation until it passes.
- For `packages/coding-agent/test/suite/`, use `test/suite/harness.ts` + the faux provider. No real provider APIs, keys, or paid tokens.
- When regressions tests for fixing a github issue, add a comment with the github issue number next to the test.
- For ad-hoc scripts, `write` them to a temp file (e.g. `/tmp`), run, edit if needed, remove when done. Don't embed multi-line scripts in `bash` commands.
- Never commit unless the user asks.
## Dependency and Install Security
- Treat npm dep and lockfile changes as reviewed code. Direct external deps stay pinned to exact versions.
- When updating `undici`, you MUST read its changelog/release notes for the target version and evaluate whether any changes may affect functionality before applying the update.
- Hydrate/update locally with `npm install --ignore-scripts`; clean/CI-style with `npm ci --ignore-scripts`. Don't run lifecycle scripts unless the user asks.
- If dep metadata changes, refresh `package-lock.json` with `npm install --package-lock-only --ignore-scripts`.
- If `packages/coding-agent/npm-shrinkwrap.json` needs regen, run `node scripts/generate-coding-agent-shrinkwrap.mjs` (verify with `--check` or `npm run check`). New deps with lifecycle scripts require review and an explicit allowlist entry in that script; never add one silently.
- Pre-commit blocks lockfile commits unless `PI_ALLOW_LOCKFILE_CHANGE=1`. Don't bypass unless the user wants the lockfile change committed.
## Git
Multiple pi sessions may be running in this cwd at the same time, each modifying different files. Git operations that touch unstaged, staged, or untracked files outside your own changes will stomp on other sessions' work. Follow these rules:
Committing:
- Only commit files YOU changed in THIS session.
- Stage explicit paths (`git add <path1> <path2>`); never `git add -A` / `git add .`.
- Before committing, run `git status` and verify you are only staging your files.
- `packages/ai/src/models.generated.ts` may always be included alongside your files.
- Message format: `{feat,fix,docs}[(ai,tui,agent,coding-agent)]: <commit message> (optionally multiple lines)`. Message is informative and concise.
Never run (destroys other agents' work or bypasses checks):
- `git reset --hard`, `git checkout .`, `git clean -fd`, `git stash`, `git add -A`, `git add .`, `git commit --no-verify`.
If rebase conflicts occur:
- Resolve conflicts only in files you modified.
- If a conflict is in a file you did not modify, abort and ask the user.
- Never force push.
## Issues and PRs
See `CONTRIBUTING.md` for the contributor gate (auto-close workflows, `lgtm`/`lgtmi`, quality bar).
When reviewing PRs:
- Do not run `gh pr checkout`, `git switch`, or otherwise move the worktree to the PR branch unless the user explicitly asks.
- Use `gh pr view`, `gh pr diff`, `gh api`, and local `git show`/`git diff` against fetched refs to inspect PR metadata, commits, and patches without changing branches.
- If you need PR file contents, fetch/read them into temporary files or use `git show <ref>:<path>` without switching branches.
When creating issues:
- Add `pkg:*` labels for affected packages (`pkg:agent`, `pkg:ai`, `pkg:coding-agent`, `pkg:tui`); use all that apply.
When posting issue/PR comments:
- Write the comment to a temp file and post with `gh issue/pr comment --body-file` (never multi-line markdown via `--body`).
- Keep comments concise, technical, in the user's tone.
- End every AI-posted comment with the AI-generated disclaimer line specified by the originating prompt (e.g. `This comment is AI-generated by `/wr``).
When closing issues via commit:
- Include `fixes #<number>` or `closes #<number>` in the message so merging auto-closes the issue. For multiple issues, repeat the keyword per issue (`closes #1, closes #2`); a shared keyword (`closes #1, #2`) only closes the first.
## Testing pi Interactive Mode with tmux
Run the TUI in a controlled terminal (from the repo root):
```bash
tmux new-session -d -s pi-test -x 80 -y 24
tmux send-keys -t pi-test "./pi-test.sh" Enter
sleep 3 && tmux capture-pane -t pi-test -p # capture after startup
tmux send-keys -t pi-test "your prompt here" Enter
tmux send-keys -t pi-test Escape # special keys (also C-o for ctrl+o, etc.)
tmux kill-session -t pi-test
```
## Changelog
Location: `packages/*/CHANGELOG.md` (one per package).
Sections under `## [Unreleased]`: `### Breaking Changes` (API changes requiring migration), `### Added`, `### Changed`, `### Fixed`, `### Removed`.
Rules:
- All new entries go under `## [Unreleased]`. Read the full section first and append to existing subsections; never duplicate them.
- Released version sections (e.g. `## [0.12.2]`) are immutable; never modify them.
- Do not create changelog entries when working on a branch other than `main` or pull request
Attribution:
- Internal (from issues): `Fixed foo bar ([#123](https://github.com/earendil-works/pi-mono/issues/123))`
- External contributions: `Added feature X ([#456](https://github.com/earendil-works/pi-mono/pull/456) by [@username](https://github.com/username))`
## Releasing
**Lockstep versioning**: all packages share one version; every release updates all together. `patch` = fixes + additions, `minor` = breaking changes. No major releases.
1. **Update CHANGELOGs**: ask the user whether they ran the `/cl` prompt on the latest commit on `main`. If not, they must run `/cl` first to audit and update each package's `[Unreleased]` section before releasing.
2. **Local smoke test**: build an unpublished release and smoke test from outside the repo (so it can't resolve workspace files):
```bash
npm run release:local -- --out /tmp/pi-local-release --force
cd /tmp
# Node package install smoke tests
/tmp/pi-local-release/node/pi --help
/tmp/pi-local-release/node/pi --version
/tmp/pi-local-release/node/pi --list-models
/tmp/pi-local-release/node/pi -p "Say exactly: ok"
/tmp/pi-local-release/node/pi
# Bun binary smoke tests
/tmp/pi-local-release/bun/pi --help
/tmp/pi-local-release/bun/pi --version
/tmp/pi-local-release/bun/pi --list-models
/tmp/pi-local-release/bun/pi -p "Say exactly: ok"
/tmp/pi-local-release/bun/pi
```
Verify both Node and Bun startup, model/account listing, interactive startup, and at least one real prompt with the intended default provider. The bare commands `/tmp/pi-local-release/node/pi` and `/tmp/pi-local-release/bun/pi` start interactive mode; run each in tmux, submit a prompt, and wait for the model reply before considering the interactive smoke test passed. Failures are release blockers unless the user explicitly accepts the risk.
3. **Run the release script**:
```bash
PI_ALLOW_LOCKFILE_CHANGE=1 npm_config_min_release_age=0 npm run release:patch # fixes + additions
PI_ALLOW_LOCKFILE_CHANGE=1 npm_config_min_release_age=0 npm run release:minor # breaking changes
```
Use `npm_config_min_release_age=0` only for the release command. The repo's normal npm age gate can otherwise block the release lockfile refresh when the current workspace package version was published recently. Review any lockfile or shrinkwrap diffs the release creates before push.
The release script bumps all package versions, updates changelogs, regenerates release artifacts, runs `npm run check`, commits `Release vX.Y.Z`, tags `vX.Y.Z`, adds fresh `## [Unreleased]` changelog sections, commits `Add [Unreleased] section for next cycle`, then pushes `main` and the tag. Do not rerun the release script after a tag was pushed.
4. **CI verifies and announces the npm release**: pushing the `vX.Y.Z` tag triggers `.github/workflows/build-binaries.yml`. The `publish-npm` job uses npm trusted publishing through GitHub Actions OIDC with environment `npm-publish`; no local `npm publish`, `npm whoami`, OTP, or WebAuthn flow is required. After publishing, `announce-pi-dev-release` verifies every public workspace package resolves at the exact release version and that its npm tarball is available, then writes the verified release marker to R2. `pi.dev/api/latest-version` reads that marker; it must never announce a release from npm before this job succeeds.
5. **If CI publish or announcement fails**: inspect the failed job. The publish helper is idempotent and skips package versions already present on npm; the announcement job rechecks availability before updating the R2 marker. Rerun the failed job or workflow after fixing CI or transient npm issues. Do not rerun `npm run release:patch` or `npm run release:minor` for the same version.
## User Override
If the user's instructions conflict with any rule in this document, ask for explicit confirmation before overriding. Only then execute their instructions.

102
third_party/pi/CONTRIBUTING.md vendored Normal file
View File

@@ -0,0 +1,102 @@
# Contributing to pi
This guide exists to save both sides time.
## Philosophy
First things first: **pi's core is minimal**.
If your feature does not belong in the core, it should be an extension. PRs that bloat the core will likely be rejected.
Pi's core exists to be minimal and to be extensible so that it can be influenced and manipulated by extensions. Even hook points for extensions however should be well considered and discussed to avoid adding unmaintainable bloat and complex interactions.
## The One Rule
**You must understand your code.** If you cannot explain what your changes do and how they interact with the rest of the system, your PR will be closed.
Using AI to write code is fine. Submitting AI-generated slop without understanding it is not.
If you use an agent, run it from the `pi` root directory so it picks up `AGENTS.md` automatically. Your agent must follow the rules and guidelines in that file.
## Contribution Gate
All issues and PRs from new contributors are auto-closed by default.
Issues submitted Friday through Sunday are not guaranteed to be reviewed. If something is urgent, ask on Discord: https://discord.com/invite/3cU7Bz4UPx
Maintainers review auto-closed issues daily and reopen worthwhile ones. Issues that do not meet the quality bar below will not be reopened or receive a reply.
Approval happens through maintainer replies on issues:
- `lgtmi`: your future issues will not be auto-closed
- `lgtm`: your future issues and PRs will not be auto-closed
The command must be at the start of the reply (optionally after one or more `@username` mentions) or at the end. `lgtmi` does not grant rights to submit PRs. Only `lgtm` grants rights to submit PRs.
## Quality Bar For Issues
If you open an issue, you must use one of the two GitHub issue templates.
If you open an issue, keep it short, concrete, and worth reading.
- Keep it concise. If it does not fit on one screen, it is too long.
- Write in your own voice (do not use an LLM to generate text, if you must, follow up with a clearly AI labeled comment).
- State the bug or request clearly.
- Explain why it matters.
- If you want to implement the change yourself, say so.
If the issue is real and written well, a maintainer may reopen it or reply with `lgtmi` or `lgtm` in the command position described above.
## Blocking
If you ignore this document twice, or if you spam the tracker with agent-generated issues, your GitHub account will be permanently blocked.
If you send a large volume of issues through automation, your GitHub account will be permanently blocked. No taksies backsies.
## Before Submitting a PR
Do not open a PR unless you have already been approved by a maintainer using `lgtm` in the command position described above.
Before submitting a PR:
```bash
npm run check
./test.sh
```
Both must pass.
Do not edit `CHANGELOG.md`. Changelog entries are added by maintainers.
If you are adding a new provider to `packages/ai`, see `AGENTS.md` for required tests.
## Questions?
Ask on [Discord](https://discord.com/invite/nKXTsAcmbT).
## FAQ
### Why are new issues and PRs auto-closed?
pi receives more issues than the maintainers can responsibly review in real time. Many reports do not meet the quality bar in this guide or do not follow CONTRIBUTING.md. Some are slung at the repository mindlessly via an agent instead of being reviewed and shaped by the person submitting them. Auto-closing creates a buffer so maintainers can review the tracker on their own schedule and reopen the issues that meet the quality bar.
### Why are weekend issues lower priority?
We triage the tracker during working hours. That means more issues can accumulate over the weekend. Anything submitted Friday through Sunday may be missed or given lower priority in the Monday review queue. If a problem is urgent, ask on Discord and include the short version, a repro, and the relevant logs.
### Why do some issues get no reply?
A reply is maintenance work too. Low-signal issues, unclear reports, duplicates, and issues that do not follow this guide may be closed without discussion. This keeps time available for reproducible bugs, thoughtful requests, and contributors who have done the work to make their report actionable.
### Why not let AI triage everything?
AI can help group duplicates, summarize reports, and spot missing information. It is not trusted to make final maintainer decisions. Polished AI-generated issues can still be wrong, misleading, or expensive to investigate. Human review remains the final gate.
### Is this hostile to contributors?
No. It is a guardrail against burnout and tracker spam. Short, concrete, reproducible issues are welcome. Thoughtful contributions are welcome. Automated slop, entitlement, and large volumes of low-effort reports are not.
## Where can I learn about plans?
Earendil uses RFCs to discuss larger changes. Not all of them are public, but
quite a few are. They can be found at [rfc.earendil.com](https://rfc.earendil.com/keyword/pi/).

21
third_party/pi/LICENSE vendored Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 Mario Zechner
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

115
third_party/pi/README.md vendored Normal file
View File

@@ -0,0 +1,115 @@
<p align="center">
<a href="https://pi.dev">
<img alt="pi logo" src="https://pi.dev/logo-auto.svg" width="128">
</a>
</p>
<p align="center">
<a href="https://discord.com/invite/3cU7Bz4UPx"><img alt="Discord" src="https://img.shields.io/badge/discord-community-5865F2?style=flat-square&logo=discord&logoColor=white" /></a>
<a href="https://www.npmjs.com/package/@earendil-works/pi-coding-agent"><img alt="npm" src="https://img.shields.io/npm/v/@earendil-works/pi-coding-agent?style=flat-square" /></a>
</p>
> New issues and PRs from new contributors are auto-closed by default. Maintainers review auto-closed issues daily. See [CONTRIBUTING.md](CONTRIBUTING.md).
# Pi Agent Harness
This is the home of the Pi agent harness project including our self extensible coding agent.
* **[@earendil-works/pi-coding-agent](packages/coding-agent)**: Interactive coding agent CLI
* **[@earendil-works/pi-agent-core](packages/agent)**: Agent runtime with tool calling and state management
* **[@earendil-works/pi-ai](packages/ai)**: Unified multi-provider LLM API (OpenAI, Anthropic, Google, …)
To learn more about Pi:
* [Visit pi.dev](https://pi.dev), the project website with demos
* [Read the documentation](https://pi.dev/docs/latest), but you can also ask the agent to explain itself
## All Packages
| Package | Description |
|---------|-------------|
| **[@earendil-works/chord](packages/chord)** | Standalone application-composition runtime for services, replicated state, RPC, and plugins |
| **[@earendil-works/pi-telemetry](packages/telemetry)** | Vendor-neutral telemetry contracts, reference adapter, conformance tests, and typed schemas |
| **[@earendil-works/pi-ai](packages/ai)** | Unified multi-provider LLM API (OpenAI, Anthropic, Google, etc.) |
| **[@earendil-works/pi-agent-core](packages/agent)** | Agent runtime with tool calling and state management |
| **[@earendil-works/pi-coding-agent](packages/coding-agent)** | Interactive coding agent CLI |
| **[@earendil-works/pi-tui](packages/tui)** | Terminal UI library with differential rendering |
For Slack/chat automation and workflows see [earendil-works/pi-chat](https://github.com/earendil-works/pi-chat).
## Permissions & Containerization
Pi does not include a built-in permission system for restricting filesystem, process, network, or credential access. By default, it runs with the permissions of the user and process that launched it.
If you need stronger boundaries, containerize or sandbox Pi. See [packages/coding-agent/docs/containerization.md](packages/coding-agent/docs/containerization.md) for three patterns:
- **Gondolin extension**: keep `pi` and provider auth on the host while routing built-in tools and `!` commands into a local Linux micro-VM.
- **Plain Docker**: run the whole `pi` process in a local container for simple isolation.
- **OpenShell**: run the whole `pi` process in a policy-controlled sandbox.
## Contributing
See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution guidelines and [AGENTS.md](AGENTS.md) for project-specific rules (for both humans and agents). Longer term plans for Pi can also be found in [RFCs](https://rfc.earendil.com/keyword/pi/).
## Development
```bash
npm install --ignore-scripts # Install all dependencies without running lifecycle scripts
npm run build # Refresh model data, then build all packages
npm run build:offline # Rebuild using existing model data without network access
npm run check # Lint, format, and type check
./test.sh # Run tests (skips LLM-dependent tests without API keys)
./pi-test.sh # Run pi from sources (can be run from any directory)
```
## Building standalone binaries from release source
GitHub releases include a versioned source archive covered by the release's `SHA256SUMS` file. Extract it and run the same build script used for the official standalone binaries:
```bash
VERSION="<release-version>"
tar -xzf "pi-${VERSION}-source.tar.gz"
cd "pi-${VERSION}"
./scripts/build-binaries.sh --offline-model-data --platform linux-x64 --out "$PWD/out"
```
The source archive includes the generated provider model data used for the release. `--offline-model-data` builds with that snapshot instead of refreshing it from live provider catalogs. The script still installs dependencies, builds the monorepo, compiles the Bun executable, and stages its runtime assets. Package maintainers who provide dependencies separately can pass `--skip-install --skip-deps`.
## Supply-chain hardening
We treat npm dependency changes as reviewed code changes.
- Direct external dependencies are pinned to exact versions. Internal workspace packages remain version-ranged.
- `.npmrc` sets `save-exact=true` and `min-release-age=2` to avoid same-day dependency releases during npm resolution.
- `package-lock.json` is the dependency ground truth. Pre-commit blocks accidental lockfile commits unless `PI_ALLOW_LOCKFILE_CHANGE=1` is set.
- `npm run check` verifies pinned direct deps, native TypeScript import compatibility, and the generated coding-agent shrinkwrap.
- The published CLI package includes `packages/coding-agent/npm-shrinkwrap.json`, generated from the root lockfile, to pin transitive deps for npm users.
- Release smoke tests use `npm run release:local` to build, pack, and create isolated npm and Bun installs outside the repo before tagging a release.
- Local release installs, documented npm installs, and `pi update --self` use `--ignore-scripts` where supported.
- CI installs with `npm ci --ignore-scripts`, and a scheduled GitHub workflow runs `npm audit --omit=dev` plus `npm audit signatures --omit=dev`.
- Shrinkwrap generation has an explicit allowlist for dependency lifecycle scripts; new lifecycle-script deps fail checks until reviewed.
## Share your OSS coding agent sessions
If you use Pi or other coding agents for open source work, please share your sessions.
Public OSS session data helps improve coding agents with real-world tasks, tool use, failures, and fixes instead of toy benchmarks.
For the full explanation, see [this post on X](https://x.com/badlogicgames/status/2037811643774652911).
To publish sessions, use [`badlogic/pi-share-hf`](https://github.com/badlogic/pi-share-hf). Read its README.md for setup instructions. All you need is a Hugging Face account, the Hugging Face CLI, and `pi-share-hf`.
You can also watch [this video](https://x.com/badlogicgames/status/2041151967695634619), where I show how I publish my `pi-mono` sessions.
I regularly publish my own `pi-mono` work sessions here:
- [badlogicgames/pi-mono on Hugging Face](https://huggingface.co/datasets/badlogicgames/pi-mono)
## License
MIT
<p align="center">
<a href="https://pi.dev">pi.dev</a> domain graciously donated by
<br /><br />
<a href="https://exe.dev"><img src="packages/coding-agent/docs/images/exy.png" alt="Exy mascot" width="48" /><br />exe.dev</a>
</p>

87
third_party/pi/SECURITY.md vendored Normal file
View File

@@ -0,0 +1,87 @@
# Security Policy
This document should guide you about understanding the security concept behind
Pi and also where the boundaries are.
In general Pi is a coding agent that runs locally within the security boundary
of the user that is running it. It's the responsibility of the user to monitor
its operations or to contain it within a container, virtual machine or other
Sandbox solution.
Pi treats the local user account and files writable by that account as inside
the same trust boundary as the Pi process itself. If an attacker can modify files
under the user's home directory, workspace, shell startup files, environment, or
Pi configuration, they can generally influence Pi or other local developer tools.
Reports that depend on such prior local write access are not security
vulnerabilities unless they demonstrate how Pi grants that write access or crosses
an operating-system privilege boundary.
Pi relies on users installing trustworthy extensions and loading trustworthy
skills and only to use pi within trusted repositories. This is because files
like `AGENTS.md` or instructions in comments can be used to prompt inject the
coding agent trivially and this cannot be protected against.
## Reporting a Vulnerability
If you believe you found a security vulnerability in pi or another package in
this repository, please report it privately by either:
- Emailing `security@earendil.com`, or
- Opening a private report through GitHub Security Advisories for this repository
Please include:
- A description of the issue and its impact
- Steps to reproduce, proof of concept, or relevant logs
- Affected package, version, commit, or configuration
- Any known mitigations
Do not open a public issue for security-sensitive reports. We will review
reports and coordinate disclosure as appropriate.
## Scope
Security issues in the distributed packages, command-line tools, APIs, and
repository code are in scope as well as earendil operated infrastructure
on `pi.dev`.
## Out Of Scope
- Local code execution or sandboxing behavior (the Pi coding agent intentionally does not have a sandbox)
- Behavior of pi extensions or skills installed by the user
- Risks from working in untrusted repositories
- Risks from installing untrusted extensions, skills, packages, or tools
- Isuses caused by non trustworthy MITM proxies
- Public internet exposure of a Pi installation
- Prompt injection attacks
- Exposed secrets that are third-party/user-controlled credentials
- Reports requiring the ability to create, modify, delete, or replace files,
directories, symlinks, environment variables, shell configuration, or other
user-controlled local state on the target machine. This includes `~/.pi`,
`~/.pi/agent/models.json`, workspace files, `AGENTS.md`, skills, extensions,
extension configuration, dotfiles, and files synchronized through NFS, roaming
profiles, or dotfile managers, unless the report shows how Pi itself grants
that access.
- Issues caused by intentionally weakened user configuration.
- Resource/DOS claims that require trusted local input/config against the pi coding agent.
- Reports about malicious model output.
- User-approved or user-initiated local actions presented as vulnerabilities.
## Notes for Reporters
The most useful reports show a current, reproducible security boundary bypass
with demonstrated impact. Reports that only show expected local-agent behavior,
prompt injection, or a malicious trusted extension/skill are not security
vulnerabilities under this model.
For example, a report showing that malicious contents written to a trusted Pi
configuration file cause Pi to execute commands, load attacker-controlled tools,
send credentials to an attacker-controlled endpoint, or otherwise change behavior
is out of scope.
When possible, include the exact affected path, package version or commit SHA,
configuration, and a proof of concept against the latest release or latest
`main`. For dependency reports, include evidence that the shipped dependency is
affected and that the issue is reachable through Pi. For exposed-secret reports,
include evidence that the credential is owned by Earendil or grants access to
Earendil-operated infrastructure or services.

41
third_party/pi/biome.json vendored Normal file
View File

@@ -0,0 +1,41 @@
{
"$schema": "https://biomejs.dev/schemas/2.3.5/schema.json",
"linter": {
"enabled": true,
"rules": {
"recommended": true,
"style": {
"noNonNullAssertion": "off",
"useConst": "error",
"useNodejsImportProtocol": "off"
},
"suspicious": {
"noExplicitAny": "off",
"noControlCharactersInRegex": "off",
"noEmptyInterface": "off"
}
}
},
"formatter": {
"enabled": true,
"formatWithErrors": false,
"indentStyle": "tab",
"indentWidth": 3,
"lineWidth": 120
},
"files": {
"includes": [
"packages/*/src/**/*.ts",
"packages/*/test/**/*.ts",
"packages/session-backends/*/src/**/*.ts",
"packages/session-backends/*/test/**/*.ts",
"packages/coding-agent/examples/**/*.ts",
"!**/node_modules/**/*",
"!**/test-sessions.ts",
"!**/models.generated.ts",
"!**/*.models.ts",
"!packages/mom/data/**/*",
"!!**/node_modules"
]
}
}

46
third_party/pi/mini-test.sh vendored Executable file
View File

@@ -0,0 +1,46 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MINI_SRC="$SCRIPT_DIR/packages/coding-agent/src/experimental/mini/main.ts"
MINI_DIST="$SCRIPT_DIR/packages/coding-agent/dist/experimental/mini/main.js"
# The session server is spawned detached and outlives the TUI, so it keeps running whatever code it
# started with. Restart it after changing anything under mini/, or the protocol will not match.
stop_server() {
pkill -f "mini/server/entry" 2>/dev/null || true
rm -f "${PI_AGENT_DIR:-$HOME/.pi/agent}/experimental/mini.sock"
}
USE_DIST=false
ARGS=()
for arg in "$@"; do
case "$arg" in
--dist) USE_DIST=true ;;
--fresh) stop_server ;;
--stop) stop_server; echo "Stopped the mini session server."; exit 0 ;;
--help)
cat <<'USAGE'
Usage: ./mini-test.sh [--dist] [--fresh] [--stop] [mini args...]
--dist run built output with plain node instead of tsx on sources
--fresh stop the detached session server first, so it picks up your changes
--stop stop the detached session server and exit
Mini args are passed through, for example:
./mini-test.sh --continue
USAGE
exit 0 ;;
*) ARGS+=("$arg") ;;
esac
done
if [[ "$USE_DIST" == "true" ]]; then
if [[ ! -f "$MINI_DIST" ]]; then
echo "No build found. Run: npm run build -w @earendil-works/pi-coding-agent" >&2
exit 1
fi
exec node "$MINI_DIST" ${ARGS[@]+"${ARGS[@]}"}
fi
exec "$SCRIPT_DIR/node_modules/.bin/tsx" --tsconfig "$SCRIPT_DIR/tsconfig.json" "$MINI_SRC" ${ARGS[@]+"${ARGS[@]}"}

5637
third_party/pi/package-lock.json generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

76
third_party/pi/package.json vendored Normal file
View File

@@ -0,0 +1,76 @@
{
"name": "pi-monorepo",
"private": true,
"type": "module",
"workspaces": [
"packages/*",
"packages/session-backends/*",
"packages/coding-agent/examples/extensions/with-deps",
"packages/coding-agent/examples/extensions/custom-provider-anthropic",
"packages/coding-agent/examples/extensions/custom-provider-gitlab-duo",
"packages/coding-agent/examples/extensions/sandbox",
"packages/coding-agent/examples/extensions/gondolin"
],
"scripts": {
"clean": "npm run clean --workspaces",
"build": "cd packages/chord && npm run build && cd ../tui && npm run build && cd ../telemetry && npm run build && cd ../ai && npm run build && cd ../agent && npm run build && cd ../session-backends/sqlite-node && npm run build && cd ../../protocol && npm run build && cd ../client && npm run build && cd ../server && npm run build && cd ../coding-agent && npm run build",
"build:offline": "cd packages/chord && npm run build && cd ../tui && npm run build && cd ../telemetry && npm run build && cd ../ai && npm run build:offline && cd ../agent && npm run build && cd ../session-backends/sqlite-node && npm run build && cd ../../protocol && npm run build && cd ../client && npm run build && cd ../server && npm run build && cd ../coding-agent && npm run build",
"check": "biome check --write --error-on-warnings . && npm run check:pinned-deps && npm run check:runtime-deps && npm run check:ts-imports && npm run check:entry-graphs && npm run check:shrinkwrap && npm run check:install-lock:coding-agent && tsgo --noEmit && npm run check:browser-smoke",
"check:browser-smoke": "node scripts/check-browser-smoke.mjs",
"check:pinned-deps": "node scripts/check-pinned-deps.mjs",
"check:runtime-deps": "node scripts/check-runtime-deps.mjs",
"check:package-install": "node scripts/coding-agent-consumer.mjs",
"check:shrinkwrap": "node scripts/generate-coding-agent-shrinkwrap.mjs --check",
"check:install-lock:coding-agent": "node scripts/generate-coding-agent-install-lock.mjs --check",
"check:ts-imports": "node scripts/check-ts-relative-imports.mjs",
"check:entry-graphs": "node scripts/check-entry-graphs.mjs",
"generate:models": "npm --prefix packages/ai run generate-models && npm --prefix packages/ai run generate-image-models",
"hydrate:model-data": "npm --prefix packages/ai run hydrate-model-data",
"check:model-data": "npm --prefix packages/ai run check:model-data",
"generate:model-catalog": "npm --prefix packages/ai run generate-model-catalog",
"diff:model-catalog": "node scripts/diff-model-catalog.mjs",
"eval": "npm run eval --workspace=@earendil-works/pi-evals --",
"check:model-catalog": "node scripts/publish-model-catalog.mjs --input .artifacts/model-catalog --dry-run",
"profile:tui": "node scripts/profile-coding-agent-node.mjs --mode tui",
"profile:rpc": "node scripts/profile-coding-agent-node.mjs --mode rpc",
"test": "npm run test:scripts && npm run test --workspaces --if-present",
"test:scripts": "node --test scripts/*.test.mjs",
"version:patch": "npm version patch --workspaces --no-git-tag-version --no-workspaces-update && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts",
"version:minor": "npm version minor --workspaces --no-git-tag-version --no-workspaces-update && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts",
"version:major": "npm version major --workspaces --no-git-tag-version --no-workspaces-update && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts",
"version:set": "npm version --workspaces --no-workspaces-update",
"prepublishOnly": "npm run clean && npm run build && npm run check",
"publish": "npm run prepublishOnly && node scripts/publish.mjs",
"publish:dry": "npm run prepublishOnly && node scripts/publish.mjs --dry-run",
"release:local": "node scripts/local-release.mjs",
"shrinkwrap:coding-agent": "node scripts/generate-coding-agent-shrinkwrap.mjs",
"install-lock:coding-agent": "node scripts/generate-coding-agent-install-lock.mjs",
"release:patch": "node scripts/release.mjs patch",
"release:minor": "node scripts/release.mjs minor",
"release:major": "node scripts/release.mjs major",
"release:fix-links": "node scripts/release-notes.mjs fix-github-releases",
"prepare": "husky"
},
"devDependencies": {
"@anthropic-ai/sandbox-runtime": "0.0.26",
"@biomejs/biome": "2.3.5",
"@types/node": "22.19.19",
"@typescript/native-preview": "7.0.0-dev.20260120.1",
"esbuild": "0.28.1",
"husky": "9.1.7",
"shx": "0.4.0",
"tsx": "4.22.1",
"typescript": "5.9.3"
},
"engines": {
"node": ">=22.19.0"
},
"version": "0.0.3",
"overrides": {
"protobufjs": "7.6.5",
"rimraf": "6.1.2",
"gaxios": {
"rimraf": "6.1.2"
}
}
}

707
third_party/pi/packages/agent/CHANGELOG.md vendored Normal file
View File

@@ -0,0 +1,707 @@
# Changelog
## [0.85.1] - 2026-09-05
## [0.85.0] - 2026-09-04
### Fixed
- Fixed proxied assistant responses dropping persisted provider-native thinking levels.
- Fixed the write tool reporting UTF-16 code-unit counts as byte counts by removing the misleading count ([#8979](https://github.com/earendil-works/pi/issues/8979)).
## [0.84.4] - 2026-08-28
### Breaking Changes
- Changed `prepareNextTurn` and `prepareNextTurnWithContext` to run only after `shouldStopAfterTurn` and queued-message checks determine that the agent loop will start another assistant turn. They no longer run after final or terminating turns; move end-of-run work to `agent_end` handling ([#6879](https://github.com/earendil-works/pi/issues/6879)).
### Fixed
- Fixed Windows `NodeExecutionEnv` aborts crashing when `taskkill.exe` is unavailable on `PATH` ([#6596](https://github.com/earendil-works/pi/issues/6596)).
### Removed
- Removed the withdrawn manual-drive configuration, action inspection methods, action outcomes, and snapshot action field from `AgentHarness`.
## [0.84.3] - 2026-08-24
### Fixed
- Fixed single-object `edit` tool inputs failing validation by accepting them as one-edit arrays ([#7835](https://github.com/earendil-works/pi/issues/7835)).
- Fixed root Markdown files such as `README.md` and `AGENTS.md` in skill directories being reported as broken skills unless they declare valid skill frontmatter ([#7805](https://github.com/earendil-works/pi/issues/7805)).
## [0.84.2] - 2026-08-14
### Fixed
- Fixed `streamProxy()` dropping finalized tool-call metadata such as OpenAI Responses namespaces ([#7709](https://github.com/earendil-works/pi/issues/7709)).
## [0.84.1] - 2026-08-07
### Added
- Added `BeforeToolCallResult.terminate` so blocked tool calls can participate in the existing batch early-termination rule ([#7715](https://github.com/earendil-works/pi/pull/7715) by [@muyiyr](https://github.com/muyiyr)).
### Fixed
- Fixed `Agent.reset()` clearing transcript and runtime state during active runs; it now rejects until the agent is idle ([#7717](https://github.com/earendil-works/pi/pull/7717) by [@wesleyzhangwq](https://github.com/wesleyzhangwq)).
## [0.84.0] - 2026-08-06
### Breaking Changes
- Replaced the legacy harness session model with the v4 lane-based `Session`, `SessionStorage`, and `SessionRepo` APIs, including durable operation records, global facts, shared sequence numbers, and tree-scoped lane views.
- Promoted the v2 session and `AgentHarness` API from the experimental entrypoint to the default package export and removed the experimental subpaths.
- Removed the legacy JSONL and in-memory repository APIs. Use the v4 `JsonlSessionRepo` or `InMemorySessionRepo`, both implementing the new `SessionRepo` contract.
- Added the required `FileSystem.renameFile()` operation to harness execution environments for atomic JSONL publication; custom file-system implementations must provide same-filesystem replacement semantics ([#7707](https://github.com/earendil-works/pi/pull/7707) by [@davidbrai](https://github.com/davidbrai)).
### Added
- Added typed AI-request and harness telemetry schemas, their combined schema tuple, callback helpers, and a generated schema reference.
- Added bounded `Session.findEntriesOnBranch()` and `findEntryOnBranch()` queries with explicit traversal, filtering, ordering, and limit options.
- Added a compile-complete `AgentHarness` v2 scaffold; unfinished operation paths reject with `HarnessNotImplemented` while durable execution is implemented.
- Added `JsonlSessionRepo`, a v4 append-only JSONL session repository with metadata validation and shared storage semantics ([#7611](https://github.com/earendil-works/pi/pull/7611) by [@davidbrai](https://github.com/davidbrai)).
- Added indexed `Session.findOpenOperations()` recovery queries and `RecordQuery.operationKind` filtering ([#7646](https://github.com/earendil-works/pi/pull/7646)).
- Added `AgentOptions.shouldStopAfterTurn` for gracefully stopping after a completed turn before queued messages or another model call are processed. See [Agent Options](README.md#agent-options) ([#7367](https://github.com/earendil-works/pi/pull/7367) by [@acmerfight](https://github.com/acmerfight)).
- Added proxy forwarding for arbitrary OpenAI-compatible `samplingParams` ([#7568](https://github.com/earendil-works/pi/pull/7568) by [@mrexodia](https://github.com/mrexodia)).
### Fixed
- Fixed Windows path handling for `NodeExecutionEnv` file basenames, recursive skill loading, and prompt template names.
- Fixed `JsonlSessionRepo` enforcing session IDs globally across working directories; IDs are now unique within each working directory.
- Fixed JSONL session forks and torn-tail repairs to publish atomically, avoiding partially written or corrupted sessions after interrupted writes ([#7707](https://github.com/earendil-works/pi/pull/7707) by [@davidbrai](https://github.com/davidbrai)).
## [0.83.0] - 2026-07-29
## [0.82.1] - 2026-07-25
## [0.82.0] - 2026-07-24
### Breaking Changes
- Replaced `AgentHarness`'s `ExecutionEnv` dependency and context-free `AgentTool` inputs with application-defined `toolContext` values and context-aware `AgentHarnessTool` definitions.
### Added
- Added context-aware `read`, `write`, `edit`, and `bash` harness tools backed by `ExecutionEnv`, including async bash execution preparation.
### Changed
- Aligned harness tool path handling, edit serialization, shell output capture, explicit non-inherited environments, and cross-platform process cleanup with coding-agent behavior.
### Fixed
- Fixed compaction and branch-summary requests to use fresh routing session IDs with prompt caching disabled where supported ([#6618](https://github.com/earendil-works/pi/pull/6618) by [@tmustier](https://github.com/tmustier)).
## [0.81.1] - 2026-07-21
### Added
- Added retry policy support and lifecycle events for compaction and branch-summary operations in `AgentHarness` ([#6901](https://github.com/earendil-works/pi/pull/6901) by [@davidbrai](https://github.com/davidbrai)).
### Fixed
- Restored the `Agent` `streamFn` option and host-configurable fallback for omitted agent-loop stream functions without reintroducing a `pi-ai/compat` dependency ([#6915](https://github.com/earendil-works/pi/issues/6915)).
## [0.81.0] - 2026-07-21
### Breaking Changes
- Changed `SessionStorage` to use `getPathToRootOrCompaction()`, require session name and statistics methods, support cursor-based entry reads, and store retained compaction tails as self-contained checkpoints ([#6594](https://github.com/earendil-works/pi/pull/6594) by [@cristinaponcela](https://github.com/cristinaponcela)).
- Moved the `uuidv7` export to `@earendil-works/pi-ai` ([#6834](https://github.com/earendil-works/pi/pull/6834) by [@xl0](https://github.com/xl0)).
- Replaced the optional `Agent` `streamFn` fallback with a required `streamFunction` and made low-level loop stream functions required, preventing `@earendil-works/pi-ai/compat` and all built-in providers from entering selective-provider bundles ([#6851](https://github.com/earendil-works/pi/issues/6851)).
### Added
- Added usage metadata to tool results, compaction entries, and branch summaries in the agent harness ([#6671](https://github.com/earendil-works/pi/pull/6671) by [@davidbrai](https://github.com/davidbrai)).
## [0.80.10] - 2026-07-16
## [0.80.9] - 2026-07-16
## [0.80.8] - 2026-07-16
## [0.80.7] - 2026-07-14
### Added
- Added `AgentToolResult.addedToolNames` propagation to `ToolResultMessage` so tools introduced by a result can be loaded from that transcript point onward ([#6474](https://github.com/earendil-works/pi-mono/pull/6474)).
## [0.80.6] - 2026-07-09
### Added
- Added the `max` model thinking level after `xhigh`.
## [0.80.5] - 2026-07-09
## [0.80.4] - 2026-07-09
### Added
- Added configurable harness session context entry transforms and custom-entry message projectors.
- Added custom metadata support in JSONL session headers ([#6417](https://github.com/earendil-works/pi/pull/6417) by [@ArcadiaLin](https://github.com/ArcadiaLin)).
- Exported `InMemorySessionStorage` and `JsonlSessionStorage` ([#6435](https://github.com/earendil-works/pi/issues/6435)).
### Fixed
- Fixed harness split-turn compaction to serialize summary requests so single-concurrency providers are not asked to run overlapping generations ([#5536](https://github.com/earendil-works/pi/issues/5536)).
- Fixed harness tool calls from length-truncated assistant messages to fail instead of waiting for missing tool results ([#6285](https://github.com/earendil-works/pi/pull/6285)).
- Fixed harness session ingestion to normalize `null` message content before context projection, avoiding crashes on lax imported transcripts ([#6343](https://github.com/earendil-works/pi/pull/6343)).
- Fixed non-positive or oversized harness shell execution timeouts to fail with a clear validation error instead of being clamped to an immediate timeout ([#6181](https://github.com/earendil-works/pi/issues/6181)).
- Fixed harness session storage short entry ids to use the random tail of the generated uuidv7 instead of the timestamp prefix, which was nearly constant between calls ([#6242](https://github.com/earendil-works/pi/issues/6242)).
## [0.80.3] - 2026-06-30
### Added
- Added `prepareNextTurnWithContext` for `Agent` users that need the next-turn loop context.
### Fixed
- Fixed `Agent.prepareNextTurn` to keep receiving the run abort signal instead of the next-turn context.
## [0.80.2] - 2026-06-23
### Changed
- Renamed the public harness shell execution options type from `ExecutionEnvExecOptions` to `ShellExecOptions`.
## [0.80.1] - 2026-06-23
## [0.80.0] - 2026-06-23
### Breaking Changes
- `AgentHarnessOptions.models` is required and is the only auth path: the harness streams turns, compaction, and branch summarization through the provided `Models` instance (`models.streamSimple()`/`completeSimple()`), resolving auth through the providers. `AgentHarnessOptions.getApiKeyAndHeaders` is removed — apps that resolved keys per request now express that as provider auth (`ApiKeyAuth`/`OAuthAuth`) on the providers in the `Models` collection. Build one with `createModels()` + provider factories (or `builtinModels()` from `@earendil-works/pi-ai/providers/all`); tests use `fauxProvider()`.
- `compact()`, `generateSummary()`, and `generateBranchSummary()` take a `Models` parameter and no longer accept explicit `apiKey`/`headers`.
- `StreamFn` is defined structurally (`(model, context, options?) => AssistantMessageEventStream | Promise<...>`); `Models.streamSimple` satisfies it.
- Removed the `@earendil-works/pi-agent-core/base` selective-provider entrypoint; use the root package with an explicit `Models` instance instead.
### Fixed
- Fixed harness session names to normalize newline characters before storing labels ([#5999](https://github.com/earendil-works/pi/pull/5999) by [@haoqixu](https://github.com/haoqixu)).
- Fixed harness compaction estimates to ignore malformed all-zero assistant usage after truncated responses ([#5526](https://github.com/earendil-works/pi/pull/5526) by [@dmmulroy](https://github.com/dmmulroy)).
## [0.79.10] - 2026-06-22
## [0.79.9] - 2026-06-20
### Fixed
- Fixed Node execution environment commands through legacy WSL `bash.exe` to pass scripts over stdin so shell variables expand in the target bash ([#5893](https://github.com/earendil-works/pi/issues/5893)).
## [0.79.8] - 2026-06-19
### Added
- Added `@earendil-works/pi-agent-core/base` for bundlers that want to pair the agent core with selective `@earendil-works/pi-ai/base` provider registration ([#5348](https://github.com/earendil-works/pi/pull/5348) by [@FredKSchott](https://github.com/FredKSchott)).
## [0.79.7] - 2026-06-18
## [0.79.6] - 2026-06-16
## [0.79.5] - 2026-06-16
## [0.79.4] - 2026-06-15
## [0.79.3] - 2026-06-13
## [0.79.2] - 2026-06-12
### Fixed
- Fixed late tool progress callbacks after tool settlement to be ignored instead of emitting stale `tool_execution_update` events ([#5573](https://github.com/earendil-works/pi/issues/5573)).
## [0.79.1] - 2026-06-09
## [0.79.0] - 2026-06-08
### Fixed
- Fixed the compaction summarization system prompt to use neutral AI assistant wording for non-coding agents ([#5401](https://github.com/earendil-works/pi/issues/5401)).
## [0.78.1] - 2026-06-04
## [0.78.0] - 2026-05-29
## [0.77.0] - 2026-05-28
### Breaking Changes
- Renamed agent harness `model_select` and `thinking_level_select` events to `model_update` and `thinking_level_update`.
### Added
- Added agent harness tool registry APIs, `tools_update` events, branch-scoped active-tool persistence, and duplicate tool validation.
## [0.76.0] - 2026-05-27
### Fixed
- Fixed context token estimates to count user image attachments consistently with tool result images ([#4983](https://github.com/earendil-works/pi/issues/4983)).
## [0.75.5] - 2026-05-23
## [0.75.4] - 2026-05-20
### Changed
- Changed source syntax to avoid TypeScript constructs that require JavaScript emit, keeping the package compatible with Node.js strip-only TypeScript checks.
- Removed the package-level development watch script now that the root TypeScript check validates strip-only-compatible sources.
### Fixed
- Fixed tool-call preflight to stop preparing sibling tool calls after the run is aborted ([#4276](https://github.com/earendil-works/pi/issues/4276)).
- Fixed tail truncation for oversized single-line output that ends with a trailing newline ([#4715](https://github.com/earendil-works/pi/issues/4715)).
- Fixed Windows Node execution environment command spawns to hide helper console windows from background processes ([#4699](https://github.com/earendil-works/pi/issues/4699)).
## [0.75.3] - 2026-05-18
## [0.75.2] - 2026-05-18
## [0.75.1] - 2026-05-18
## [0.75.0] - 2026-05-17
### Breaking Changes
- Raised the minimum supported Node.js version to 22.19.0.
## [0.74.1] - 2026-05-16
## [0.74.0] - 2026-05-07
## [0.73.1] - 2026-05-07
## [0.73.0] - 2026-05-04
## [0.72.1] - 2026-05-02
### Changed
- Changed the default agent transport to `auto` so providers can use their best available transport by default ([#4083](https://github.com/badlogic/pi-mono/issues/4083)).
## [0.72.0] - 2026-05-01
### Added
- Added `shouldStopAfterTurn` to the low-level agent loop config for gracefully exiting after a completed turn before polling queued messages or starting another LLM call.
## [0.71.1] - 2026-05-01
## [0.71.0] - 2026-04-30
## [0.70.6] - 2026-04-28
## [0.70.5] - 2026-04-27
## [0.70.4] - 2026-04-27
## [0.70.3] - 2026-04-27
## [0.70.2] - 2026-04-24
## [0.70.1] - 2026-04-24
## [0.70.0] - 2026-04-23
## [0.69.0] - 2026-04-22
### Breaking Changes
- Migrated public TypeBox-facing types and examples from `@sinclair/typebox` 0.34.x to `typebox` 1.x. Install and import from `typebox` instead of relying on `@sinclair/typebox` transitively ([#3112](https://github.com/badlogic/pi-mono/issues/3112))
### Added
- Added `terminate: true` tool-result hints to skip the automatic follow-up LLM call when every finalized tool result in the current batch opts into early termination ([#3525](https://github.com/badlogic/pi-mono/issues/3525))
## [0.68.1] - 2026-04-22
### Fixed
- Fixed `streamProxy()` to preserve the proxy-safe serializable subset of stream options, including session, transport, retry-delay, metadata, header, cache-retention, and thinking-budget settings ([#3512](https://github.com/badlogic/pi-mono/issues/3512))
- Fixed parallel tool execution to emit `tool_execution_end` as soon as each tool is finalized, while still emitting persisted tool-result messages in assistant source order ([#3503](https://github.com/badlogic/pi-mono/issues/3503))
## [0.68.0] - 2026-04-20
### Changed
- Clarified parallel tool execution ordering docs to specify that final tool lifecycle and tool-result artifacts are emitted in tool completion order.
## [0.67.68] - 2026-04-17
## [0.67.67] - 2026-04-17
### Fixed
- Fixed parallel tool-call finalization to convert `afterToolCall` hook throws into error tool results instead of aborting the batch ([#3084](https://github.com/badlogic/pi-mono/issues/3084))
## [0.67.6] - 2026-04-16
## [0.67.5] - 2026-04-16
## [0.67.4] - 2026-04-16
## [0.67.3] - 2026-04-15
## [0.67.2] - 2026-04-14
## [0.67.1] - 2026-04-13
## [0.67.0] - 2026-04-13
## [0.66.1] - 2026-04-08
## [0.66.0] - 2026-04-08
## [0.65.2] - 2026-04-06
## [0.65.1] - 2026-04-05
## [0.65.0] - 2026-04-03
### Breaking Changes
- `AgentState` has been reshaped:
- `streamMessage` was renamed to `streamingMessage`
- `error` was renamed to `errorMessage`
- `isStreaming`, `streamingMessage`, `pendingToolCalls`, and `errorMessage` are now readonly in the public API
- `pendingToolCalls` is now typed as `ReadonlySet<string>`
- `tools` and `messages` are now accessor properties, and assigning either field copies the provided top-level array instead of preserving array identity
- `AgentOptions.initialState` no longer accepts runtime-owned fields. Remove `isStreaming`, `streamingMessage`, `pendingToolCalls`, and `errorMessage` from `initialState` values.
- Removed `Agent` mutator methods in favor of direct property access:
- `agent.setSystemPrompt(value)` -> `agent.state.systemPrompt = value`
- `agent.setModel(model)` -> `agent.state.model = model`
- `agent.setThinkingLevel(level)` -> `agent.state.thinkingLevel = level`
- `agent.setTools(tools)` -> `agent.state.tools = tools`
- `agent.replaceMessages(messages)` -> `agent.state.messages = messages`
- `agent.appendMessage(message)` -> `agent.state.messages.push(message)`
- `agent.clearMessages()` -> `agent.state.messages = []`
- `agent.setToolExecution(mode)` -> `agent.toolExecution = mode`
- `agent.setBeforeToolCall(fn)` -> `agent.beforeToolCall = fn`
- `agent.setAfterToolCall(fn)` -> `agent.afterToolCall = fn`
- `agent.setTransport(transport)` -> `agent.transport = transport`
- Removed queue mode getter/setter methods in favor of properties:
- `agent.setSteeringMode(mode)` -> `agent.steeringMode = mode`
- `agent.getSteeringMode()` -> `agent.steeringMode`
- `agent.setFollowUpMode(mode)` -> `agent.followUpMode = mode`
- `agent.getFollowUpMode()` -> `agent.followUpMode`
- `Agent.subscribe()` listeners are now awaited and receive the active `AbortSignal`:
- `agent.subscribe((event) => { ... })` -> `agent.subscribe(async (event, signal) => { ... })`
- `agent_end` is now the final emitted event for a run, but not the idle boundary
- `agent.waitForIdle()`, `agent.prompt(...)`, and `agent.continue()` now settle only after awaited `agent_end` listeners finish
- `agent.state.isStreaming` remains `true` until that settlement completes
## [0.64.0] - 2026-03-29
### Added
- Added `AgentTool.prepareArguments` hook to prepare raw tool call arguments before schema validation, enabling compatibility shims for resumed sessions with outdated tool schemas
## [0.63.2] - 2026-03-29
### Added
- Added `Agent.signal` to expose the active abort signal for the current turn, allowing callers to forward cancellation into nested async work ([#2660](https://github.com/badlogic/pi-mono/issues/2660))
## [0.63.1] - 2026-03-27
## [0.63.0] - 2026-03-27
## [0.62.0] - 2026-03-23
## [0.61.1] - 2026-03-20
## [0.61.0] - 2026-03-20
## [0.60.0] - 2026-03-18
## [0.59.0] - 2026-03-17
## [0.58.4] - 2026-03-16
### Fixed
- Fixed steering messages to wait until the current assistant message's tool-call batch fully finishes instead of skipping pending tool calls.
## [0.58.3] - 2026-03-15
## [0.58.2] - 2026-03-15
## [0.58.1] - 2026-03-14
## [0.58.0] - 2026-03-14
### Added
- Added `beforeToolCall` and `afterToolCall` hooks to `AgentOptions` and `AgentLoopConfig` for preflight blocking and post-execution tool result mutation.
### Changed
- Added configurable tool execution mode to `Agent` and `agentLoop` via `toolExecution: "parallel" | "sequential"`, with `parallel` as the default. Parallel mode preflights tool calls sequentially, executes allowed tools concurrently, and emits final tool results in assistant source order.
## [0.57.1] - 2026-03-07
## [0.57.0] - 2026-03-07
## [0.56.3] - 2026-03-06
## [0.56.2] - 2026-03-05
## [0.56.1] - 2026-03-05
## [0.56.0] - 2026-03-04
## [0.55.4] - 2026-03-02
## [0.55.3] - 2026-02-27
## [0.55.2] - 2026-02-27
## [0.55.1] - 2026-02-26
## [0.55.0] - 2026-02-24
## [0.54.2] - 2026-02-23
## [0.54.1] - 2026-02-22
## [0.54.0] - 2026-02-19
## [0.53.1] - 2026-02-19
## [0.53.0] - 2026-02-17
## [0.52.12] - 2026-02-13
### Added
- Added `transport` to `AgentOptions` and `AgentLoopConfig` forwarding, allowing stream transport preference (`"sse"`, `"websocket"`, `"auto"`) to flow into provider calls.
## [0.52.11] - 2026-02-13
## [0.52.10] - 2026-02-12
## [0.52.9] - 2026-02-08
## [0.52.8] - 2026-02-07
## [0.52.7] - 2026-02-06
### Fixed
- Fixed `continue()` to resume queued steering/follow-up messages when context currently ends in an assistant message, and preserved one-at-a-time steering ordering during assistant-tail resumes ([#1312](https://github.com/badlogic/pi-mono/pull/1312) by [@ferologics](https://github.com/ferologics))
## [0.52.6] - 2026-02-05
## [0.52.5] - 2026-02-05
## [0.52.4] - 2026-02-05
## [0.52.3] - 2026-02-05
## [0.52.2] - 2026-02-05
## [0.52.1] - 2026-02-05
## [0.52.0] - 2026-02-05
## [0.51.6] - 2026-02-04
## [0.51.5] - 2026-02-04
## [0.51.4] - 2026-02-03
## [0.51.3] - 2026-02-03
## [0.51.2] - 2026-02-03
## [0.51.1] - 2026-02-02
## [0.51.0] - 2026-02-01
## [0.50.9] - 2026-02-01
## [0.50.8] - 2026-02-01
### Added
- Added `maxRetryDelayMs` option to `AgentOptions` to cap server-requested retry delays. Passed through to the underlying stream function. ([#1123](https://github.com/badlogic/pi-mono/issues/1123))
## [0.50.7] - 2026-01-31
## [0.50.6] - 2026-01-30
## [0.50.5] - 2026-01-30
## [0.50.3] - 2026-01-29
## [0.50.2] - 2026-01-29
## [0.50.1] - 2026-01-26
## [0.50.0] - 2026-01-26
## [0.49.3] - 2026-01-22
## [0.49.2] - 2026-01-19
## [0.49.1] - 2026-01-18
## [0.49.0] - 2026-01-17
## [0.48.0] - 2026-01-16
## [0.47.0] - 2026-01-16
## [0.46.0] - 2026-01-15
## [0.45.7] - 2026-01-13
## [0.45.6] - 2026-01-13
## [0.45.5] - 2026-01-13
## [0.45.4] - 2026-01-13
## [0.45.3] - 2026-01-13
## [0.45.2] - 2026-01-13
## [0.45.1] - 2026-01-13
## [0.45.0] - 2026-01-13
## [0.44.0] - 2026-01-12
## [0.43.0] - 2026-01-11
## [0.42.5] - 2026-01-11
## [0.42.4] - 2026-01-10
## [0.42.3] - 2026-01-10
## [0.42.2] - 2026-01-10
## [0.42.1] - 2026-01-09
## [0.42.0] - 2026-01-09
## [0.41.0] - 2026-01-09
## [0.40.1] - 2026-01-09
## [0.40.0] - 2026-01-08
## [0.39.1] - 2026-01-08
## [0.39.0] - 2026-01-08
## [0.38.0] - 2026-01-08
### Added
- `thinkingBudgets` option on `Agent` and `AgentOptions` to customize token budgets per thinking level ([#529](https://github.com/badlogic/pi-mono/pull/529) by [@melihmucuk](https://github.com/melihmucuk))
## [0.37.8] - 2026-01-07
## [0.37.7] - 2026-01-07
## [0.37.6] - 2026-01-06
## [0.37.5] - 2026-01-06
## [0.37.4] - 2026-01-06
## [0.37.3] - 2026-01-06
### Added
- `sessionId` option on `Agent` to forward session identifiers to LLM providers for session-based caching.
## [0.37.2] - 2026-01-05
## [0.37.1] - 2026-01-05
## [0.37.0] - 2026-01-05
### Fixed
- `minimal` thinking level now maps to `minimal` reasoning effort instead of being treated as `low`.
## [0.36.0] - 2026-01-05
## [0.35.0] - 2026-01-05
## [0.34.2] - 2026-01-04
## [0.34.1] - 2026-01-04
## [0.34.0] - 2026-01-04
## [0.33.0] - 2026-01-04
## [0.32.3] - 2026-01-03
## [0.32.2] - 2026-01-03
## [0.32.1] - 2026-01-03
## [0.32.0] - 2026-01-03
### Breaking Changes
- **Queue API replaced with steer/followUp**: The `queueMessage()` method has been split into two methods with different delivery semantics ([#403](https://github.com/badlogic/pi-mono/issues/403)):
- `steer(msg)`: Interrupts the agent mid-run. Delivered after current tool execution, skips remaining tools.
- `followUp(msg)`: Waits until the agent finishes. Delivered only when there are no more tool calls or steering messages.
- **Queue mode renamed**: `queueMode` option renamed to `steeringMode`. Added new `followUpMode` option. Both control whether messages are delivered one-at-a-time or all at once.
- **AgentLoopConfig callbacks renamed**: `getQueuedMessages` split into `getSteeringMessages` and `getFollowUpMessages`.
- **Agent methods renamed**:
- `queueMessage()` → `steer()` and `followUp()`
- `clearMessageQueue()` → `clearSteeringQueue()`, `clearFollowUpQueue()`, `clearAllQueues()`
- `setQueueMode()`/`getQueueMode()` → `setSteeringMode()`/`getSteeringMode()` and `setFollowUpMode()`/`getFollowUpMode()`
### Fixed
- `prompt()` and `continue()` now throw if called while the agent is already streaming, preventing race conditions and corrupted state. Use `steer()` or `followUp()` to queue messages during streaming, or `await` the previous call.
## [0.31.1] - 2026-01-02
## [0.31.0] - 2026-01-02
### Breaking Changes
- **Transport abstraction removed**: `ProviderTransport`, `AppTransport`, and `AgentTransport` interface have been removed. Use the `streamFn` option directly for custom streaming implementations.
- **Agent options renamed**:
- `transport` → removed (use `streamFn` instead)
- `messageTransformer` → `convertToLlm`
- `preprocessor` → `transformContext`
- **`AppMessage` renamed to `AgentMessage`**: All references to `AppMessage` have been renamed to `AgentMessage` for consistency.
- **`CustomMessages` renamed to `CustomAgentMessages`**: The declaration merging interface has been renamed.
- **`UserMessageWithAttachments` and `Attachment` types removed**: Attachment handling is now the responsibility of the `convertToLlm` function.
- **Agent loop moved from `@mariozechner/pi-ai`**: The `agentLoop`, `agentLoopContinue`, and related types have moved to this package. Import from `@mariozechner/pi-agent-core` instead.
### Added
- `streamFn` option on `Agent` for custom stream implementations. Default uses `streamSimple` from pi-ai.
- `streamProxy()` utility function for browser apps that need to proxy LLM calls through a backend server. Replaces the removed `AppTransport`.
- `getApiKey` option for dynamic API key resolution (useful for expiring OAuth tokens like GitHub Copilot).
- `agentLoop()` and `agentLoopContinue()` low-level functions for running the agent loop without the `Agent` class wrapper.
- New exported types: `AgentLoopConfig`, `AgentContext`, `AgentTool`, `AgentToolResult`, `AgentToolUpdateCallback`, `StreamFn`.
### Changed
- `Agent` constructor now has all options optional (empty options use defaults).
- `queueMessage()` is now synchronous (no longer returns a Promise).

517
third_party/pi/packages/agent/README.md vendored Normal file
View File

@@ -0,0 +1,517 @@
# @earendil-works/pi-agent-core
Stateful agent with tool execution and event streaming. Built on `@earendil-works/pi-ai`.
## Installation
```bash
npm install @earendil-works/pi-agent-core
```
### SQLite session backends
The SQLite session backend and the `node:sqlite` adapter live in a separate package, `@earendil-works/pi-session-backend-sqlite-node`, so the core package does not pull in runtime builtins or native SQLite dependencies by default. The backend accepts a runtime-specific SQLite factory, allowing other session backends to ship as their own packages in the future.
## Quick Start
```typescript
import { Agent } from "@earendil-works/pi-agent-core";
import { createModels } from "@earendil-works/pi-ai";
import { anthropicProvider } from "@earendil-works/pi-ai/providers/anthropic";
const models = createModels();
models.setProvider(anthropicProvider());
const model = models.getModel("anthropic", "claude-sonnet-4-6");
if (!model) throw new Error("Model not found");
const agent = new Agent({
initialState: {
systemPrompt: "You are a helpful assistant.",
model,
},
streamFn: models.streamSimple.bind(models),
});
agent.subscribe((event) => {
if (event.type === "message_update" && event.assistantMessageEvent.type === "text_delta") {
// Stream just the new text chunk
process.stdout.write(event.assistantMessageEvent.delta);
}
});
await agent.prompt("Hello!");
```
## Experimental facet services
Transport-neutral facet-service primitives live in `@earendil-works/chord`. The agent core does not export the service runtime.
## Core Concepts
### AgentMessage vs LLM Message
The agent works with `AgentMessage`, a flexible type that can include:
- Standard LLM messages (`user`, `assistant`, `toolResult`)
- Custom app-specific message types via declaration merging
LLMs only understand `user`, `assistant`, and `toolResult`. The `convertToLlm` function bridges this gap by filtering and transforming messages before each LLM call.
### Message Flow
```
AgentMessage[] → transformContext() → AgentMessage[] → convertToLlm() → Message[] → LLM
(optional) (required)
```
1. **transformContext**: Prune old messages, inject external context
2. **convertToLlm**: Filter out UI-only messages, convert custom types to LLM format
## Event Flow
The agent emits events for UI updates. Understanding the event sequence helps build responsive interfaces.
### prompt() Event Sequence
When you call `prompt("Hello")`:
```
prompt("Hello")
├─ agent_start
├─ turn_start
├─ message_start { message: userMessage } // Your prompt
├─ message_end { message: userMessage }
├─ message_start { message: assistantMessage } // LLM starts responding
├─ message_update { message: partial... } // Streaming chunks
├─ message_update { message: partial... }
├─ message_end { message: assistantMessage } // Complete response
├─ turn_end { message, toolResults: [] }
└─ agent_end { messages: [...] }
```
### With Tool Calls
If the assistant calls tools, the loop continues:
```
prompt("Read config.json")
├─ agent_start
├─ turn_start
├─ message_start/end { userMessage }
├─ message_start { assistantMessage with toolCall }
├─ message_update...
├─ message_end { assistantMessage }
├─ tool_execution_start { toolCallId, toolName, args }
├─ tool_execution_update { partialResult } // If tool streams
├─ tool_execution_end { toolCallId, result }
├─ message_start/end { toolResultMessage }
├─ turn_end { message, toolResults: [toolResult] }
│
├─ turn_start // Next turn
├─ message_start { assistantMessage } // LLM responds to tool result
├─ message_update...
├─ message_end
├─ turn_end
└─ agent_end
```
Tool execution mode is configurable:
- `parallel` (default): preflight tool calls sequentially, execute allowed tools concurrently, emit `tool_execution_end` as soon as each tool is finalized, then emit toolResult messages and `turn_end.toolResults` in assistant source order
- `sequential`: execute tool calls one by one, matching the historical behavior
In parallel mode, tool completion events follow tool completion order, but persisted toolResult messages still follow assistant source order.
The mode can be set globally via `toolExecution` in the agent config, or per-tool via `executionMode` on `AgentTool`. If any tool call in a batch targets a tool with `executionMode: "sequential"`, the entire batch executes sequentially regardless of the global setting.
The `beforeToolCall` hook runs after `tool_execution_start` and validated argument parsing. It can block execution and attach `terminate: true` to the blocked result. The `afterToolCall` hook runs after tool execution finishes and before `tool_execution_end` and final tool result message events are emitted.
Tools, blocked `beforeToolCall` results, and `afterToolCall` overrides can return `terminate: true` to hint that the automatic follow-up LLM call should be skipped. The loop only stops early when every finalized tool result in that batch sets `terminate: true`. Mixed batches continue normally.
The `Agent` class accepts `shouldStopAfterTurn` in `AgentOptions`. Low-level loop callers can set the same hook in `AgentLoopConfig`:
```typescript
const stream = agentLoop(
prompts,
context,
{
model,
convertToLlm,
shouldStopAfterTurn: async ({ message, toolResults, context, newMessages }) => {
return shouldCompactBeforeNextTurn(context.messages);
},
},
undefined,
models.streamSimple.bind(models),
);
```
`shouldStopAfterTurn` runs after `turn_end` is emitted and after the assistant response and any tool executions have completed normally. If it returns `true`, the loop emits `agent_end` and exits before polling steering or follow-up queues, and before starting another LLM call. It does not abort the provider stream, does not cancel running tools, and does not alter the assistant message stop reason. The `AgentOptions` callback also receives the active run's `AbortSignal` as its second argument.
When you use the `Agent` class, assistant `message_end` processing is treated as a barrier before tool preflight begins. That means `beforeToolCall` sees agent state that already includes the assistant message that requested the tool call.
### continue() Event Sequence
`continue()` resumes from existing context without adding a new message. Use it for retries after errors.
```typescript
// After an error, retry from current state
await agent.continue();
```
The last message in context must be `user` or `toolResult` (not `assistant`).
### Event Types
| Event | Description |
|-------|-------------|
| `agent_start` | Agent begins processing |
| `agent_end` | Final event for the run. Awaited subscribers for this event still count toward settlement |
| `turn_start` | New turn begins (one LLM call + tool executions) |
| `turn_end` | Turn completes with assistant message and tool results |
| `message_start` | Any message begins (user, assistant, toolResult) |
| `message_update` | **Assistant only.** Includes `assistantMessageEvent` with delta |
| `message_end` | Message completes |
| `tool_execution_start` | Tool begins |
| `tool_execution_update` | Tool streams progress |
| `tool_execution_end` | Tool completes |
`Agent.subscribe()` listeners are awaited in registration order. `agent_end` means no more loop events will be emitted, but `await agent.waitForIdle()` and `await agent.prompt(...)` only settle after awaited `agent_end` listeners finish.
## Agent Options
```typescript
const agent = new Agent({
// Initial state
initialState: {
systemPrompt: string,
model: Model<any>,
thinkingLevel: "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max",
tools: AgentTool<any>[],
messages: AgentMessage[],
},
// Convert AgentMessage[] to LLM Message[] (required for custom message types)
convertToLlm: (messages) => messages.filter(...),
// Transform context before convertToLlm (for pruning, compaction)
transformContext: async (messages, signal) => pruneOldMessages(messages),
// Steering mode: "one-at-a-time" (default) or "all"
steeringMode: "one-at-a-time",
// Follow-up mode: "one-at-a-time" (default) or "all"
followUpMode: "one-at-a-time",
// Required stream function
streamFn: models.streamSimple.bind(models),
// Session ID for provider caching
sessionId: "session-123",
// Dynamic API key resolution (for expiring OAuth tokens)
getApiKey: async (provider) => refreshToken(),
// Tool execution mode: "parallel" (default) or "sequential"
toolExecution: "parallel",
// Preflight each tool call after args are validated. Can block execution.
beforeToolCall: async ({ toolCall, args, context }) => {
if (toolCall.name === "bash") {
return { block: true, reason: "bash is disabled", terminate: true };
}
},
// Postprocess each tool result before final tool events are emitted.
afterToolCall: async ({ toolCall, result, isError, context }) => {
if (toolCall.name === "notify_done" && !isError) {
return { terminate: true };
}
if (!isError) {
return { details: { ...result.details, audited: true } };
}
},
// Stop gracefully after a completed turn, before queued messages are polled.
shouldStopAfterTurn: async ({ context }, signal) => {
return shouldCompactBeforeNextTurn(context.messages, signal);
},
// Custom thinking budgets for token-based providers
thinkingBudgets: {
minimal: 128,
low: 512,
medium: 1024,
high: 2048,
},
});
```
## Agent State
```typescript
interface AgentState {
systemPrompt: string;
model: Model<any>;
thinkingLevel: ThinkingLevel;
tools: AgentTool<any>[];
messages: AgentMessage[];
readonly isStreaming: boolean;
readonly streamingMessage?: AgentMessage;
readonly pendingToolCalls: ReadonlySet<string>;
readonly errorMessage?: string;
}
```
Access state via `agent.state`.
Assigning `agent.state.tools = [...]` or `agent.state.messages = [...]` copies the top-level array before storing it. Mutating the returned array mutates the current agent state.
During streaming, `agent.state.streamingMessage` contains the current partial assistant message.
`agent.state.isStreaming` remains `true` until the run fully settles, including awaited `agent_end` subscribers.
## Methods
### Prompting
```typescript
// Text prompt
await agent.prompt("Hello");
// With images
await agent.prompt("What's in this image?", [
{ type: "image", data: base64Data, mimeType: "image/jpeg" }
]);
// AgentMessage directly
await agent.prompt({ role: "user", content: "Hello", timestamp: Date.now() });
// Continue from current context (last message must be user or toolResult)
await agent.continue();
```
### State Management
```typescript
agent.state.systemPrompt = "New prompt";
agent.state.model = getModel("openai", "gpt-4o");
agent.state.thinkingLevel = "medium";
agent.state.tools = [myTool];
agent.toolExecution = "sequential";
agent.beforeToolCall = async ({ toolCall }) => undefined;
agent.afterToolCall = async ({ toolCall, result }) => undefined;
agent.shouldStopAfterTurn = async ({ context }) => shouldCompactBeforeNextTurn(context.messages);
agent.state.messages = newMessages; // top-level array is copied
agent.state.messages.push(message);
agent.reset();
```
### Session and Thinking Budgets
```typescript
agent.sessionId = "session-123";
agent.thinkingBudgets = {
minimal: 128,
low: 512,
medium: 1024,
high: 2048,
};
```
### Control
```typescript
agent.abort(); // Cancel current operation
await agent.waitForIdle(); // Wait for completion
```
### Events
```typescript
const unsubscribe = agent.subscribe(async (event, signal) => {
if (event.type === "agent_end") {
// Final barrier work for the run
await flushSessionState(signal);
}
});
unsubscribe();
```
## Steering and Follow-up
Steering messages let you interrupt the agent while tools are running. Follow-up messages let you queue work after the agent would otherwise stop.
```typescript
agent.steeringMode = "one-at-a-time";
agent.followUpMode = "one-at-a-time";
// While agent is running tools
agent.steer({
role: "user",
content: "Stop! Do this instead.",
timestamp: Date.now(),
});
// After the agent finishes its current work
agent.followUp({
role: "user",
content: "Also summarize the result.",
timestamp: Date.now(),
});
const steeringMode = agent.steeringMode;
const followUpMode = agent.followUpMode;
agent.clearSteeringQueue();
agent.clearFollowUpQueue();
agent.clearAllQueues();
```
Use clearSteeringQueue, clearFollowUpQueue, or clearAllQueues to drop queued messages.
When steering messages are detected after a turn completes:
1. All tool calls from the current assistant message have already finished
2. Steering messages are injected
3. The LLM responds on the next turn
Follow-up messages are checked only when there are no more tool calls and no steering messages. If any are queued, they are injected and another turn runs.
## Custom Message Types
Extend `AgentMessage` via declaration merging:
```typescript
declare module "@earendil-works/pi-agent-core" {
interface CustomAgentMessages {
notification: { role: "notification"; text: string; timestamp: number };
}
}
// Now valid
const msg: AgentMessage = { role: "notification", text: "Info", timestamp: Date.now() };
```
Handle custom types in `convertToLlm`:
```typescript
const agent = new Agent({
streamFn: models.streamSimple.bind(models),
convertToLlm: (messages) => messages.flatMap(m => {
if (m.role === "notification") return []; // Filter out
return [m];
}),
});
```
## Tools
Define tools using `AgentTool`:
```typescript
import { Type } from "typebox";
const readFileTool: AgentTool = {
name: "read_file",
label: "Read File", // For UI display
description: "Read a file's contents",
parameters: Type.Object({
path: Type.String({ description: "File path" }),
}),
// Override execution mode for this tool (optional).
// "sequential" forces the entire batch to run one at a time.
// "parallel" allows concurrent execution with other tool calls.
// If omitted, the global toolExecution config applies.
executionMode: "sequential",
execute: async (toolCallId, params, signal, onUpdate) => {
const content = await fs.readFile(params.path, "utf-8");
// Optional: stream progress
onUpdate?.({ content: [{ type: "text", text: "Reading..." }], details: {} });
// Optional: add `terminate: true` here to skip the automatic follow-up LLM call
// when every finalized tool result in the batch does the same.
return {
content: [{ type: "text", text: content }],
details: { path: params.path, size: content.length },
};
},
};
agent.state.tools = [readFileTool];
```
### Error Handling
**Throw an error** when a tool fails. Do not return error messages as content.
```typescript
execute: async (toolCallId, params, signal, onUpdate) => {
if (!fs.existsSync(params.path)) {
throw new Error(`File not found: ${params.path}`);
}
// Return content only on success
return { content: [{ type: "text", text: "..." }] };
}
```
Thrown errors are caught by the agent and reported to the LLM as tool errors with `isError: true`.
Return `terminate: true` from `execute()`, a blocked `beforeToolCall`, or `afterToolCall` to hint that the agent should stop after the current tool batch. This only takes effect when every finalized tool result in the batch is terminating. The hint is runtime-only; emitted `toolResult` transcript messages remain standard LLM tool results.
## Proxy Usage
For browser apps that proxy through a backend:
```typescript
import { Agent, streamProxy } from "@earendil-works/pi-agent-core";
const agent = new Agent({
streamFn: (model, context, options) =>
streamProxy(model, context, {
...options,
authToken: "...",
proxyUrl: "https://your-server.com",
}),
});
```
## Low-Level API
For direct control without the Agent class:
```typescript
import { agentLoop, agentLoopContinue } from "@earendil-works/pi-agent-core";
const context: AgentContext = {
systemPrompt: "You are helpful.",
messages: [],
tools: [],
};
const config: AgentLoopConfig = {
model: getModel("openai", "gpt-4o"),
convertToLlm: (msgs) => msgs.filter(m => ["user", "assistant", "toolResult"].includes(m.role)),
toolExecution: "parallel", // overridden by per-tool executionMode if set
beforeToolCall: async ({ toolCall, args, context }) => undefined,
afterToolCall: async ({ toolCall, result, isError, context }) => undefined,
};
const userMessage = { role: "user", content: "Hello", timestamp: Date.now() };
const streamFn = models.streamSimple.bind(models);
for await (const event of agentLoop([userMessage], context, config, undefined, streamFn)) {
console.log(event.type);
}
// Continue from existing context
for await (const event of agentLoopContinue(context, config, undefined, streamFn)) {
console.log(event.type);
}
```
These low-level streams are observational. They preserve event order, but they do not wait for your async event handling to settle before later producer phases continue. If you need message processing to act as a barrier before tool preflight, use the `Agent` class instead of raw `agentLoop()` or `agentLoopContinue()`.
## License
MIT

34
third_party/pi/packages/agent/benchmark/session/README.md vendored Normal file
View File

@@ -0,0 +1,34 @@
# Session benchmarks
Run these commands from `packages/agent`.
These benchmarks use deterministic synthetic data and make no claim to reproduce production traffic. Storage and repository fork datasets are linear branches of user-message entries with fixed 256-byte text payloads, deterministic IDs, and 250-entry seed transactions. Repository catalog datasets contain deterministic closed sessions.
## Timing
The timing suite runs scenarios against every target in `storage-targets.ts` and `session-repo-targets.ts`. Read scenarios reuse one immutable fixture per backend/dataset and validate their result once before timing. Every warmup and measured write runs once against an independently prepared fixture, so entry counts, sequence numbers, and durable artifacts start in equivalent states. Implementations are registered under the same Vitest suite so additional backends are directly comparable.
Storage writes cover a single message, a 100-message transaction, and a mixed message/register/usage append to a 1k-entry synthetic branch. Repository scenarios cover creating an empty session, opening or deleting a closed empty session, listing 100, 1k, and 10k closed sessions, and forking the current branch of open 1k- and 10k-entry source sessions. Fixture preparation, transaction generation, and validation happen outside the measured callback.
```sh
npm run bench:session:timing
npm run bench:session:timing -- -t "scan latest"
```
## Loaded footprint
The memory profile starts a fresh Node.js process with forced GC for every backend/dataset pair. Its baseline is recorded before generating and ingesting the dataset, so the result includes retained payload data and backend structures. Temporary generation data should be collected before the final reading, while RSS can still reflect allocator growth.
```sh
npm run bench:session:memory
```
## Storage allocation sampling
The allocation profile prebuilds all transactions, then samples allocations made while committing them. This excludes synthetic payload generation and reports estimated cumulative allocated bytes, including objects later collected, plus the largest sampled allocation sites. V8 sampling is approximate and does not report an exact object count.
```sh
npm run bench:session:allocations
```
Timing results are not CI performance gates. Run them on an otherwise idle machine and compare results produced on the same hardware and Node.js version.

57
third_party/pi/packages/agent/benchmark/session/allocation-profile-worker.ts vendored Normal file
View File

@@ -0,0 +1,57 @@
import type { HeapProfiler } from "node:inspector";
import { Session } from "node:inspector/promises";
import {
STORAGE_BENCHMARK_DATASETS,
generateStorageBenchmarkSeedTransactions,
} from "../../src/harness/session/testing/index.ts";
import { storageBenchmarkTargets } from "./storage-targets.ts";
const SAMPLING_INTERVAL_BYTES = 4_096;
const TOP_SITE_COUNT = 5;
function collectAllocationSites(node: HeapProfiler.SamplingHeapProfileNode, sites: Map<string, number>): number {
const { functionName, url, lineNumber, columnNumber } = node.callFrame;
const site = `${functionName || "(anonymous)"} (${url || "native"}:${lineNumber + 1}:${columnNumber + 1})`;
if (node.selfSize > 0) sites.set(site, (sites.get(site) ?? 0) + node.selfSize);
let total = node.selfSize;
for (const child of node.children) total += collectAllocationSites(child, sites);
return total;
}
const [targetName, datasetName] = process.argv.slice(2);
const target = storageBenchmarkTargets.find((candidate) => candidate.name === targetName);
if (target === undefined) throw new Error(`Unknown storage benchmark target: ${targetName}`);
const dataset = STORAGE_BENCHMARK_DATASETS.find((candidate) => candidate.name === datasetName);
if (dataset === undefined) throw new Error(`Unknown storage benchmark dataset: ${datasetName}`);
// Inputs and the empty fixture exist before sampling so the profile measures
// allocation caused by Storage.commit(), not synthetic payload generation.
const transactions = [...generateStorageBenchmarkSeedTransactions(dataset)];
await using fixture = await target.createFixture();
const inspector = new Session();
inspector.connect();
try {
await inspector.post("HeapProfiler.startSampling", {
samplingInterval: SAMPLING_INTERVAL_BYTES,
includeObjectsCollectedByMajorGC: true,
includeObjectsCollectedByMinorGC: true,
});
for (const transaction of transactions) await fixture.storage.commit(transaction);
const { profile } = await inspector.post("HeapProfiler.stopSampling");
const sites = new Map<string, number>();
const allocatedBytes = collectAllocationSites(profile.head, sites);
console.log(
JSON.stringify({
target: target.name,
dataset: dataset.name,
entryCount: dataset.entryCount,
allocatedBytes,
topSites: [...sites.entries()]
.map(([site, bytes]) => ({ site, bytes }))
.sort((left, right) => right.bytes - left.bytes)
.slice(0, TOP_SITE_COUNT),
}),
);
} finally {
inspector.disconnect();
}

55
third_party/pi/packages/agent/benchmark/session/allocation-profile.ts vendored Normal file
View File

@@ -0,0 +1,55 @@
import { execFile } from "node:child_process";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { STORAGE_BENCHMARK_DATASETS } from "../../src/harness/session/testing/index.ts";
import { storageBenchmarkTargets } from "./storage-targets.ts";
interface AllocationSiteResult {
readonly site: string;
readonly bytes: number;
}
interface AllocationProfileResult {
readonly target: string;
readonly dataset: string;
readonly entryCount: number;
readonly allocatedBytes: number;
readonly topSites: readonly AllocationSiteResult[];
}
const execFileAsync = promisify(execFile);
const packageRoot = fileURLToPath(new URL("../..", import.meta.url));
const workerPath = fileURLToPath(new URL("./allocation-profile-worker.ts", import.meta.url));
const tsconfigPath = fileURLToPath(new URL("../tsconfig.json", import.meta.url));
const results: AllocationProfileResult[] = [];
for (const target of storageBenchmarkTargets) {
for (const dataset of STORAGE_BENCHMARK_DATASETS) {
const { stdout } = await execFileAsync(
process.execPath,
["--import", "tsx", workerPath, target.name, dataset.name],
{ cwd: packageRoot, env: { ...process.env, TSX_TSCONFIG_PATH: tsconfigPath } },
);
results.push(JSON.parse(stdout) as AllocationProfileResult);
}
}
const mebibytes = (bytes: number): string => (bytes / 1024 / 1024).toFixed(2);
console.log("Storage allocation sampling (inputs generated before sampling):");
console.table(
results.map((result) => ({
backend: result.target,
dataset: result.dataset,
"allocated MiB": mebibytes(result.allocatedBytes),
"allocated bytes/entry": Math.round(result.allocatedBytes / result.entryCount),
})),
);
for (const result of results) {
console.log(`\n${result.target}, ${result.dataset}, top sampled allocation sites:`);
console.table(
result.topSites.map(({ site, bytes }) => ({
site,
"sampled MiB": mebibytes(bytes),
})),
);
}

179
third_party/pi/packages/agent/benchmark/session/benchmark.ts vendored Normal file
View File

@@ -0,0 +1,179 @@
import { strictEqual } from "node:assert/strict";
import { bench, describe } from "vitest";
export interface BenchmarkTarget<TFixture extends AsyncDisposable> {
readonly name: string;
createFixture(): Promise<TFixture>;
}
const READ_BENCHMARK_OPTIONS = {
time: 500,
iterations: 10,
warmupTime: 100,
warmupIterations: 5,
} as const;
const WRITE_BENCHMARK_ITERATIONS = 30;
const WRITE_BENCHMARK_WARMUP_ITERATIONS = 5;
const WRITE_BENCHMARK_OPTIONS = {
time: 0,
iterations: WRITE_BENCHMARK_ITERATIONS,
warmupTime: 0,
warmupIterations: WRITE_BENCHMARK_WARMUP_ITERATIONS,
} as const;
interface NamedBenchmarkDataset {
readonly name: string;
}
interface ReadBenchmarkScenario<TSubject, TDataset> {
readonly name: string;
expectedResult(dataset: TDataset): number;
run(subject: TSubject, dataset: TDataset): Promise<number>;
}
interface RegisterReadBenchmarksOptions<
TDataset extends NamedBenchmarkDataset,
TFixture extends AsyncDisposable,
TSubject,
> {
readonly datasets: readonly TDataset[];
readonly targets: readonly BenchmarkTarget<TFixture>[];
readonly scenarios: readonly ReadBenchmarkScenario<TSubject, TDataset>[];
prepare(fixture: TFixture, dataset: TDataset): Promise<void>;
getSubject(fixture: TFixture): TSubject;
}
interface PreparedReadFixture<TFixture> {
readonly datasetName: string;
readonly targetName: string;
readonly fixture: TFixture;
}
interface NamedBenchmarkScenario {
readonly name: string;
}
interface RegisterWriteBenchmarksOptions<
TScenario extends NamedBenchmarkScenario,
TFixture extends AsyncDisposable,
TSubject,
> {
readonly targets: readonly BenchmarkTarget<TFixture>[];
readonly scenarios: readonly TScenario[];
prepare(fixture: TFixture, scenario: TScenario): Promise<TSubject>;
expectedResult(scenario: TScenario): number;
run(subject: TSubject, scenario: TScenario): Promise<number>;
}
interface PreparedWriteFixtures<TSubject> {
readonly scenarioName: string;
readonly targetName: string;
readonly pendingSubjects: TSubject[];
}
/** Prepares, validates, registers, and disposes one immutable fixture per target and dataset. */
export async function registerReadBenchmarks<
TDataset extends NamedBenchmarkDataset,
TFixture extends AsyncDisposable,
TSubject,
>(options: RegisterReadBenchmarksOptions<TDataset, TFixture, TSubject>): Promise<void> {
const fixtures: TFixture[] = [];
const preparedFixtures: PreparedReadFixture<TFixture>[] = [];
process.once("beforeExit", async () => {
await Promise.all(fixtures.map((fixture) => fixture[Symbol.asyncDispose]()));
});
for (const dataset of options.datasets) {
for (const target of options.targets) {
const fixture = await target.createFixture();
fixtures.push(fixture);
await options.prepare(fixture, dataset);
const subject = options.getSubject(fixture);
for (const scenario of options.scenarios) {
strictEqual(await scenario.run(subject, dataset), scenario.expectedResult(dataset));
}
preparedFixtures.push({ datasetName: dataset.name, targetName: target.name, fixture });
}
}
for (const dataset of options.datasets) {
for (const scenario of options.scenarios) {
describe(`${scenario.name} (${dataset.name})`, () => {
for (const target of options.targets) {
const prepared = preparedFixtures.find(
(candidate) => candidate.datasetName === dataset.name && candidate.targetName === target.name,
);
if (prepared === undefined) throw new Error("Benchmark fixture was not initialized");
const subject = options.getSubject(prepared.fixture);
bench(
target.name,
async () => {
await scenario.run(subject, dataset);
},
READ_BENCHMARK_OPTIONS,
);
}
});
}
}
}
/** Prepares, validates, registers, and disposes one equivalent fixture per write invocation. */
export async function registerWriteBenchmarks<
TScenario extends NamedBenchmarkScenario,
TFixture extends AsyncDisposable,
TSubject,
>(options: RegisterWriteBenchmarksOptions<TScenario, TFixture, TSubject>): Promise<void> {
const fixtures: TFixture[] = [];
const preparedFixtures: PreparedWriteFixtures<TSubject>[] = [];
process.once("beforeExit", async () => {
await Promise.all(fixtures.map((fixture) => fixture[Symbol.asyncDispose]()));
});
for (const scenario of options.scenarios) {
for (const target of options.targets) {
const validationFixture = await target.createFixture();
fixtures.push(validationFixture);
const validationSubject = await options.prepare(validationFixture, scenario);
strictEqual(await options.run(validationSubject, scenario), options.expectedResult(scenario));
const pendingSubjects: TSubject[] = [];
for (
let index = 0;
index < WRITE_BENCHMARK_WARMUP_ITERATIONS + WRITE_BENCHMARK_ITERATIONS;
index++
) {
const fixture = await target.createFixture();
fixtures.push(fixture);
pendingSubjects.push(await options.prepare(fixture, scenario));
}
preparedFixtures.push({ scenarioName: scenario.name, targetName: target.name, pendingSubjects });
}
}
for (const scenario of options.scenarios) {
describe(scenario.name, () => {
for (const target of options.targets) {
const prepared = preparedFixtures.find(
(candidate) => candidate.scenarioName === scenario.name && candidate.targetName === target.name,
);
if (prepared === undefined) throw new Error("Benchmark fixtures were not initialized");
bench(
target.name,
async () => {
const subject = prepared.pendingSubjects.shift();
if (subject === undefined) throw new Error("Benchmark fixture pool was exhausted");
await options.run(subject, scenario);
},
WRITE_BENCHMARK_OPTIONS,
);
}
});
}
}

34
third_party/pi/packages/agent/benchmark/session/loaded-footprint-worker.ts vendored Normal file
View File

@@ -0,0 +1,34 @@
import {
STORAGE_BENCHMARK_DATASETS,
seedStorageBenchmark,
} from "../../src/harness/session/testing/index.ts";
import { storageBenchmarkTargets } from "./storage-targets.ts";
function collectGarbage(): void {
if (globalThis.gc === undefined) throw new Error("Loaded footprint measurement requires Node.js --expose-gc");
for (let index = 0; index < 3; index++) globalThis.gc();
}
const [targetName, datasetName] = process.argv.slice(2);
const target = storageBenchmarkTargets.find((candidate) => candidate.name === targetName);
if (target === undefined) throw new Error(`Unknown storage benchmark target: ${targetName}`);
const dataset = STORAGE_BENCHMARK_DATASETS.find((candidate) => candidate.name === datasetName);
if (dataset === undefined) throw new Error(`Unknown storage benchmark dataset: ${datasetName}`);
await using fixture = await target.createFixture();
collectGarbage();
const before = process.memoryUsage();
await seedStorageBenchmark(fixture.storage, dataset);
collectGarbage();
const after = process.memoryUsage();
console.log(
JSON.stringify({
target: target.name,
dataset: dataset.name,
entryCount: dataset.entryCount,
heapUsedBytes: after.heapUsed - before.heapUsed,
rssBytes: after.rss - before.rss,
externalBytes: after.external - before.external,
}),
);

51
third_party/pi/packages/agent/benchmark/session/loaded-footprint.ts vendored Normal file
View File

@@ -0,0 +1,51 @@
import { execFile } from "node:child_process";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { STORAGE_BENCHMARK_DATASETS } from "../../src/harness/session/testing/index.ts";
import { storageBenchmarkTargets } from "./storage-targets.ts";
interface LoadedFootprintResult {
readonly target: string;
readonly dataset: string;
readonly entryCount: number;
readonly heapUsedBytes: number;
readonly rssBytes: number;
readonly externalBytes: number;
}
const execFileAsync = promisify(execFile);
const packageRoot = fileURLToPath(new URL("../..", import.meta.url));
const workerPath = fileURLToPath(new URL("./loaded-footprint-worker.ts", import.meta.url));
const tsconfigPath = fileURLToPath(new URL("../tsconfig.json", import.meta.url));
const results: LoadedFootprintResult[] = [];
for (const target of storageBenchmarkTargets) {
for (const dataset of STORAGE_BENCHMARK_DATASETS) {
const { stdout } = await execFileAsync(
process.execPath,
[
"--expose-gc",
"--import",
"tsx",
workerPath,
target.name,
dataset.name,
],
{ cwd: packageRoot, env: { ...process.env, TSX_TSCONFIG_PATH: tsconfigPath } },
);
results.push(JSON.parse(stdout) as LoadedFootprintResult);
}
}
const mebibytes = (bytes: number): string => (bytes / 1024 / 1024).toFixed(2);
console.log("Loaded footprint (synthetic payloads generated and ingested after the baseline):");
console.table(
results.map((result) => ({
backend: result.target,
dataset: result.dataset,
"heap MiB": mebibytes(result.heapUsedBytes),
"heap bytes/entry": Math.round(result.heapUsedBytes / result.entryCount),
"RSS MiB": mebibytes(result.rssBytes),
"external MiB": mebibytes(result.externalBytes),
})),
);

22
third_party/pi/packages/agent/benchmark/session/session-repo-targets.ts vendored Normal file
View File

@@ -0,0 +1,22 @@
import { MemorySessionRepo } from "../../src/harness/session/index.ts";
import type { SessionRepo } from "../../src/harness/session/types.ts";
import type { BenchmarkTarget } from "./benchmark.ts";
export interface SessionRepoBenchmarkFixture extends AsyncDisposable {
readonly repo: SessionRepo;
}
const NOW = 1_700_000_000_000;
export const sessionRepoBenchmarkTargets = [
{
name: "memory",
createFixture() {
const repo = new MemorySessionRepo({ now: () => NOW });
return Promise.resolve({
repo,
[Symbol.asyncDispose]: () => repo.close(),
});
},
},
] satisfies readonly BenchmarkTarget<SessionRepoBenchmarkFixture>[];

62
third_party/pi/packages/agent/benchmark/session/session-repo.bench.ts vendored Normal file
View File

@@ -0,0 +1,62 @@
import {
SESSION_REPO_CATALOG_BENCHMARK_DATASETS,
SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS,
SESSION_REPO_CATALOG_WRITE_BENCHMARK_SCENARIOS,
SESSION_REPO_FORK_BENCHMARK_DATASETS,
SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS,
seedSessionRepoCatalogBenchmark,
seedSessionRepoForkBenchmark,
} from "../../src/harness/session/testing/index.ts";
import { registerReadBenchmarks, registerWriteBenchmarks } from "./benchmark.ts";
import { sessionRepoBenchmarkTargets } from "./session-repo-targets.ts";
await registerReadBenchmarks({
datasets: SESSION_REPO_CATALOG_BENCHMARK_DATASETS,
targets: sessionRepoBenchmarkTargets,
scenarios: SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS,
async prepare(fixture, dataset) {
await seedSessionRepoCatalogBenchmark(fixture.repo, dataset);
},
getSubject(fixture) {
return fixture.repo;
},
});
await registerWriteBenchmarks({
targets: sessionRepoBenchmarkTargets,
scenarios: SESSION_REPO_CATALOG_WRITE_BENCHMARK_SCENARIOS,
prepare(fixture, scenario) {
return scenario.prepare(fixture.repo);
},
expectedResult(scenario) {
return scenario.expectedResult;
},
run(operation) {
return operation.run();
},
});
const forkBenchmarks = SESSION_REPO_FORK_BENCHMARK_DATASETS.flatMap((dataset) =>
SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS.map((scenario) => ({
name: `${scenario.name} (${dataset.name})`,
dataset,
scenario,
})),
);
await registerWriteBenchmarks({
targets: sessionRepoBenchmarkTargets,
scenarios: forkBenchmarks,
async prepare(fixture, benchmark) {
return {
repo: fixture.repo,
source: await seedSessionRepoForkBenchmark(fixture.repo, benchmark.dataset),
};
},
expectedResult(benchmark) {
return benchmark.scenario.expectedResult(benchmark.dataset);
},
run(subject, benchmark) {
return benchmark.scenario.run(subject.repo, subject.source, benchmark.dataset);
},
});

18
third_party/pi/packages/agent/benchmark/session/storage-targets.ts vendored Normal file
View File

@@ -0,0 +1,18 @@
import { MemoryStorage } from "../../src/harness/session/memory.ts";
import type { StorageFixture } from "../../src/harness/session/testing/index.ts";
import type { BenchmarkTarget } from "./benchmark.ts";
const NOW = 1_700_000_000_000;
export const storageBenchmarkTargets = [
{
name: "memory",
createFixture() {
const storage = new MemoryStorage({ now: () => NOW });
return Promise.resolve({
storage,
[Symbol.asyncDispose]: () => storage.close(),
});
},
},
] satisfies readonly BenchmarkTarget<StorageFixture>[];

35
third_party/pi/packages/agent/benchmark/session/storage.bench.ts vendored Normal file
View File

@@ -0,0 +1,35 @@
import {
STORAGE_BENCHMARK_DATASETS,
STORAGE_READ_BENCHMARK_SCENARIOS,
STORAGE_WRITE_BENCHMARK_SCENARIOS,
seedStorageBenchmark,
} from "../../src/harness/session/testing/index.ts";
import { registerReadBenchmarks, registerWriteBenchmarks } from "./benchmark.ts";
import { storageBenchmarkTargets } from "./storage-targets.ts";
await registerReadBenchmarks({
datasets: STORAGE_BENCHMARK_DATASETS,
targets: storageBenchmarkTargets,
scenarios: STORAGE_READ_BENCHMARK_SCENARIOS,
prepare(fixture, dataset) {
return seedStorageBenchmark(fixture.storage, dataset);
},
getSubject(fixture) {
return fixture.storage;
},
});
await registerWriteBenchmarks({
targets: storageBenchmarkTargets,
scenarios: STORAGE_WRITE_BENCHMARK_SCENARIOS,
async prepare(fixture, scenario) {
await scenario.prepare?.(fixture.storage);
return fixture.storage;
},
expectedResult(scenario) {
return scenario.writeCount;
},
run(storage, scenario) {
return scenario.run(storage);
},
});

12
third_party/pi/packages/agent/benchmark/tsconfig.json vendored Normal file
View File

@@ -0,0 +1,12 @@
{
"extends": "../../../tsconfig.base.json",
"compilerOptions": {
"baseUrl": "../../..",
"noEmit": true,
"paths": {
"@earendil-works/pi-ai": ["packages/ai/src/index.ts"],
"@earendil-works/pi-ai/*": ["packages/ai/src/*"]
}
},
"include": ["./**/*.ts", "../src/**/*.ts"]
}

355
third_party/pi/packages/agent/docs/assistant-durability.md vendored Normal file
View File

@@ -0,0 +1,355 @@
# Assistant partial durability — implementation handoff
This document specifies durable partial assistant messages for ordinary assistant generation and deferred-response polling. It builds on:
- bound typed value/list addresses from `values.md`;
- `AssistantMessageFrame`, `AssistantMessageFrameEncoder`, and `reduceAssistantMessageFrames()` from `@earendil-works/pi-ai`;
- the assistant intent/effect/settlement state machine in `harness.md`.
The design persists compact replayable stream frames without making them operation-state authority and without storing a growing full partial message on every update.
## Goals
1. Reconstruct the latest committed partial assistant message after process loss.
2. Preserve provider stream order without repeated full-snapshot write amplification.
3. Avoid provider backpressure from storage commits.
4. Reuse pi-ai's canonical frame conversion and reduction semantics.
5. Preserve current public assistant event ordering.
6. Keep operation `effect_pending` state authoritative for recovery.
7. Delete all partial frames atomically with normal or synthetic response settlement.
## Non-goals
- Inferring provider completion from frames.
- Persisting terminal `done`/`error` events separately from response settlement.
- Exactly-once provider requests.
- Public frame cursors or a frame-persistence event.
- Generic batching, timers, coalescing, or flush APIs.
- Persisting structural summary-generation streams.
- Persisting arbitrary provider SDK events or repeated full `partial` snapshots.
## Storage
Built-in address constructor in `session/values.ts`:
```ts
export const pendingAssistantFrames = (
operationId: string,
responseEntryId: string,
) => list<AssistantMessageFrame>(
"pi.pending.assistant_frame",
`${operationId}:${responseEntryId}`,
);
```
The procedure binds one exact address for ordinary generation or a deferred poll:
```ts
const frames = pendingAssistantFrames(operationId, responseEntryId);
```
`responseEntryId` is already reserved in assistant/deferred `effect_pending` state. No frame count, cursor, or list identity is stored in operation state, and later list operations receive only `frames`—never another key.
Each list element is one `AssistantMessageFrame`. The storage transaction's global write sequence orders frames. The list is auxiliary:
- missing is valid;
- it does not prove request admission, completion, success, or failure;
- it never selects the restart point;
- base restore does not read it.
## Frame contract
Create one pi-ai encoder per provider stream and feed it every event in order:
```ts
const encoder = new AssistantMessageFrameEncoder();
const frame = encoder.encode(event);
```
`partial` is the provider's shared live response-so-far helper, not an event-time snapshot. The encoder keeps per-open-block counters and trims text/thinking delta prefixes already represented by an advanced block-start snapshot. It temporarily buffers only the raw JSON prefix needed to synchronize an already-advanced tool call, then emits one checkpoint and resumes compact deltas. It never clones the growing full message on every event.
- `start` produces an empty-content metadata frame;
- a non-terminal event produces zero or one frame;
- covered queued deltas produce no frame;
- terminal `done` and `error` produce no frame because final response settlement is separate;
- a setup `error` before `start` is valid and produces no frames;
- text/thinking/tool end frames contain the authoritative completed block value;
- completed tool-call arguments remain unvalidated against a tool schema.
Do not define a second harness frame codec or reducer. Persistence stores the exported pi-ai value directly; hydration calls `reduceAssistantMessageFrames()`.
Provider event blocks may interleave. Encoding and reduction rely on `contentIndex`, never block contiguity. Text and ordinary thinking blocks must be empty when `*_start` is published and then append only through matching deltas until end; redacted thinking may be complete at start and emit no deltas. Streaming tool calls start with empty arguments and emit their complete raw JSON through deltas; a provider that starts with complete arguments must emit a cumulative delta prefix that parses to that snapshot at an event boundary before later argument deltas.
## Fresh stream scheduling
The simplest scheduling is deliberate: one list-append transaction per converted frame, enqueued without awaiting storage in the provider loop.
For every `start` or update event:
```text
encode the event against the per-stream frame encoder
→ when a frame is returned, synchronously enqueue invocation-fenced appendList(frames, frame)
→ attach the ordinary harness-fault observer to the returned promise
→ replace process-local latestFrameWrite promise reference
→ emit and await the existing message_start/message_update event
→ consume the next provider event
```
`appendList()` is called synchronously for each returned frame, so all frame mutations enter the Session line in provider-event order. The procedure does not await each write. Every promise is immediately observed for fault propagation before the latest reference replaces its predecessor. Covered queued events allocate no durable frame or write. Encoder state is proportional to open block count plus the unsynchronized prefix of an active tool-call JSON stream; it retains no second full assistant message. Model output limits bound queued frame bytes to bounded output plus frame/transaction overhead.
The event side retains existing behavior: `AssistantStreamObserver.start/update` awaits `events.emit()` for every event. There is no separate latest event-delivery promise because no assistant event delivery remains outstanding when the provider loop advances.
When the provider stream settles:
```text
stop frame admission
→ await latestFrameWrite when present
→ run after_response
→ emit message_end
→ classify and commit final response settlement
```
The Session mutation line is FIFO, so completion of `latestFrameWrite` implies every earlier append completed. There is no array of promises, timer, batcher, active/waiting state, coalescer, or public/internal flush method.
A failed frame append faults the harness before `after_response` starts. The complete final response remains process-local and does not commit after a storage fault.
## Normal settlement
Every assistant response settlement that used a frame list deletes that exact list in the same transaction as the immutable response entry, usage, tip, and next operation state:
```text
TX[
insert response entry R,
insert usage U,
setValue(branchTip(lane), R),
deleteList(frames),
setValue(operationState(operationId), classified next state)
]
```
This applies to:
- successful assistant responses;
- provider `error`/`aborted` responses;
- valid deferred responses;
- deferred poll responses.
`after_response` may transform the final response before settlement. Frames preserve provider-stream observation, while the immutable entry remains the post-hook canonical result.
A crash after the final frame append but before settlement still restores `effect_pending`; frames do not turn a complete-looking draft into a settled response.
## Unknown-outcome generation recovery
An orphaned assistant generation `effect_pending` has no surviving provider stream. Activation:
1. constructs `frames = pendingAssistantFrames(operationId, responseEntryId)` from current typed state and reads bounded pages from that exact address;
2. reduces frame values with `reduceAssistantMessageFrames()`;
3. constructs a harness-owned synthetic assistant response under the already-reserved response ID;
4. preserves reconstructed partial content and safe message identity metadata when frames exist;
5. sets `stopReason: "error"`, zero usage, and an explicit interruption/unknown-outcome `errorMessage`/diagnostic;
6. commits the synthetic response, zero-usage row, frame-list deletion, tip, and ordinary retry/failure state atomically.
Required warning meaning:
```text
The provider request was interrupted. The preceding content is the latest
committed partial response; newer live output may be missing, and the external
request outcome is unknown.
```
If no start frame committed, the harness constructs the same synthetic error with empty content from the captured model/API identity.
The synthetic response follows ordinary assistant error classification:
- attempts remain → insert the error response and enter the ordinary retry-wait/next-attempt path;
- cap reached → insert it and terminal-fail the operation in the same settlement transaction.
Error responses remain durable transcript history but are omitted from later provider context by the existing projection rule. Partial tool calls inside the interrupted error response never execute.
Recovery does not run `after_response`: there is no trustworthy complete provider result to transform.
## Cancellation
A live cancelled provider stream settles through its ordinary final `aborted` response. All accepted frames are awaited first, and normal settlement deletes the list.
For restored cancelled assistant/deferred `effect_pending`, cancellation reconciliation:
- reduces available frames;
- constructs a synthetic `aborted` response preserving the committed partial content;
- uses zero usage and the existing reserved IDs;
- atomically inserts the response and deletes the frame list;
- starts no provider request and runs no response hook.
Cancellation wins classification even if reduced content appears complete.
## Deferred polling
A deferred poll that returns an `AssistantMessageEventStream` uses the same `pendingAssistantFrames(operationId, responseEntryId)` address constructor.
- normal pending/ready/error settlement deletes the poll's frame list;
- restored cancellation synthesizes an aborted response from frames;
- an unknown restored poll without a permit remains suspended and may expose its durable partial in snapshots;
- when a poll permit replaces the unknown poll with fresh reserved response/usage IDs, that intent transaction deletes the abandoned old frame-list address;
- the replacement poll starts a fresh list under its fresh response ID.
Frames never change poll-number rules.
## Structural generation scope
Structural summary-generation streams remain process-local. They do not emit public assistant-message lifecycle and may span multiple nested provider requests before one structural publication. Existing attempt-level retry and usage recovery remains authoritative.
Do not store their intermediate text at a `pendingAssistantFrames(...)` address in this slice. If structural partial diagnostics become a requirement, add a separate explicitly scoped consumer rather than silently reusing transcript-assistant semantics.
## Snapshots and reconnect
`LaneSnapshot.streamingMessage` means the latest observed partial assistant message, not proof that a provider stream is currently attached.
Precedence:
1. newest process-local partial while a live stream is owned;
2. otherwise, for assistant/deferred `effect_pending`, the value reduced from committed frame pages;
3. otherwise absent.
A restored lane may therefore be `suspended` with a non-undefined `streamingMessage`. The field remains outside `transcript`; only `entry_added` moves a complete response into transcript history and clears the partial.
Snapshot hydration constructs the exact bound frame address from trusted typed operation state and enforces the assistant consumer's total frame/page budget. It does not scan arbitrary lists or perform a broad semantic restore audit.
Reconnect replays no historical `message_start` or `message_update` events. The snapshot carries the durable partial. Recovery later emits the ordinary recovery-tagged synthetic message lifecycle for the response it actually settles.
## Events
A started generation retains this live event ordering:
```text
message_start
→ message_update* each listener delivery awaited by provider loop
→ await latest frame write
→ after_response
→ message_end
→ atomic response settlement + frame-list delete
→ entry_added
→ usage
```
A request setup failure may produce `error` before `start`; that path emits no `message_start` or frame and proceeds through `after_response`, `message_end`, and ordinary error settlement. Successful `done` and updates before `start` are protocol defects.
Frame commits emit ordinary storage telemetry only. There is no public frame event and no claim that a `message_update` was durable. `entry_added` remains the only proof that the final assistant entry committed.
A crash may occur after a live update event but before its asynchronously queued frame append commits. Reconnect then shows the latest committed frame prefix, which may be older than the last live event.
## Close, fault, and external finalization
Close is a controlled crash:
- no synthetic response is written;
- already-enqueued frame commits are ordinary admitted session work and may finish under the close barrier;
- process loss may discard mutations that had not committed;
- reopen restores the latest committed frame prefix under unchanged `effect_pending` state.
A frame-commit storage failure faults the harness. No later response settlement commits in that process.
Authorized external finalization deletes the operation-owned frame-list address in its terminal transaction. Every append mutation verifies current operation/response ownership on the Session line:
- append first → external terminal cleanup deletes the list;
- finalization first → stale append rejects without recreating state.
## Terminal cleanup, forks, and migrations
Normal/synthetic response settlement should already delete its exact frame address. The operation terminal transaction also defensively constructs and deletes the current operation-owned frame address when state is assistant/deferred `effect_pending`.
Idle forks never copy lists in the `pi.pending.assistant_frame` address family. Precise rewrites and migrations page frame lists and preserve element sequences when retaining them.
A migration changing `AssistantMessageFrame` shape must map every surviving element or explicitly delete the whole list and leave `effect_pending` recovery with no partial. It must never infer completion from legacy frames.
JSONL retains deleted frame bytes until snapshot compaction. Logical deletion is immediate.
## Races
| Race | Required result |
|---|---|
| frame append vs next frame | synchronous lane enqueue preserves provider-event order |
| frame append vs stream settlement | settlement awaits the latest promise; all accepted appends finish first |
| live update event vs frame commit | either may finish first; event is observation and reconnect uses only committed frames |
| append vs external finalization | append first is deleted by cleanup; finalization first fences the append |
| process loss with queued writes | only the committed prefix restores |
| final frame vs response settlement | frame commits first; settlement atomically deletes the list and inserts final entry |
| activation vs snapshot | both reduce the same committed sequence prefix; activation may then settle and clear it |
| unknown generation vs retry | synthetic partial error commits under old reserved IDs before later attempt starts |
| unknown deferred poll vs replacement | old list is deleted with fresh replacement intent; new response ID gets a new list |
## Invariants
1. Scalar assistant/deferred state is the sole restart authority.
2. One effect-pending response ID constructs exactly one assistant frame-list address.
3. Every stored element is an exported pi-ai `AssistantMessageFrame`.
4. Terminal `done`/`error` events are never stored as frames.
5. Frame order is a subsequence of provider event order; zero-frame covered events do not disturb order.
6. Awaiting the latest frame-write promise at stream settlement implies all accepted appends completed.
7. Frames never establish provider completion or suppress unknown-outcome recovery.
8. Final or synthetic response settlement atomically deletes the exact frame list.
9. A restored partial may appear in `streamingMessage` but never in `transcript` before settlement.
10. Interrupted partial tool calls never produce a tool plan because the synthetic response stops with `error`/`aborted`.
11. Structural generation never writes a `pendingAssistantFrames(...)` list in this slice.
12. Terminal cleanup, external finalization, and idle forks leave no operation-owned assistant frame list.
## Required tests
### Frame integration
- one per-stream encoder handles shared live partials and synchronous event bursts without duplicate content;
- each event appends zero or one frame, with covered queued deltas appending nothing;
- `done`/`error` append nothing, including pre-generation error;
- interleaved content indexes preserve sequence;
- provider loop does not await individual frame writes;
- frame appends enqueue synchronously before the next provider event;
- only the latest promise reference is retained;
- awaiting the latest promise implies all earlier writes completed;
- bounded output bounds queued frame memory;
- storage failure prevents `after_response` and faults the harness.
### Settlement and recovery
- every ordinary response class deletes the frame list atomically;
- crash at every frame/settlement boundary;
- no frames and partial text/thinking/tool-call frames;
- authoritative end-frame content survives;
- interrupted generation commits partial synthetic error then retries or fails at cap;
- interrupted partial tool calls never execute;
- recovery uses zero usage and existing reserved IDs;
- cancellation preserves committed partial content in synthetic aborted response;
- `after_response` never runs for restored synthetic settlement.
### Deferred, snapshots, and lifecycle
- deferred poll frame persistence and normal cleanup;
- unknown poll snapshot without permit;
- replacement intent deletes abandoned poll frames;
- live partial takes precedence over durable reduction;
- reopen exposes reduced `streamingMessage` on suspended effect-pending lane;
- no historical update event replay;
- recovery settlement clears partial on `entry_added`;
- structural generation writes no assistant frame list.
### Storage lifecycle
- Memory/JSONL/SQLite reduce identical frame sequences;
- frame list is absent after normal, synthetic, cancellation, and external terminal transitions;
- idle fork excludes frames;
- JSONL compaction removes deleted frame bytes;
- migration maps or explicitly discards every legacy frame;
- instrumentation records append/delete ordering without frame content in telemetry.
## Implementation map
Expected runtime areas:
- built-in `pendingAssistantFrames(operationId, responseEntryId)` address constructor in `session/values.ts`;
- bound value/list storage implementation from `values.md`;
- assistant execution observer and generation procedure;
- deferred polling procedure;
- activation and cancellation recovery;
- snapshot hydration;
- terminal cleanup, forks, and migrations;
- instrumented writer and backend conformance tests.
Implement bound typed value/list addresses first, then frame enqueue/settlement, then recovery/snapshots. Update `harness.md` with this complete lifecycle before implementing runtime assistant parity.

1468
third_party/pi/packages/agent/docs/harness.md vendored Normal file
View File

File diff suppressed because it is too large Load Diff

214
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/FINDINGS.md vendored Normal file
View File

@@ -0,0 +1,214 @@
# 01-delta: known defects and measured findings
Everything here was found **after** `delta.md` and the prototype implementation beside it were written. None of it is applied to the code in this directory. Production now lives in `packages/chord/src/delta/index.ts`: flush-time dirty tracking resolves D1, and production remeasurement closes D2 without a new API. See [`append-decision.md`](append-decision.md). Each item below remains the original problem, evidence, and candidate-fix record.
Every number was measured with **`node --experimental-strip-types`**, not `tsx`.
That matters: tsx transpiles through esbuild and inflated the same benchmark by
**2.6x** (183 µs vs 63 µs per write). Do not benchmark this module through a
transpiler.
---
## D1. Interleaved paths defeat coalescing (correctness-adjacent, unbounded memory)
**Severity: high.** This is the one to fix first.
### Problem
`flush()` coalesces only **adjacent** ops. A producer that alternates between two
paths therefore never coalesces anything, and the ops array grows with the number
of writes rather than with the size of the value.
This is not hypothetical. It is exactly what bash does: it updates the output text
and a byte counter on every chunk.
```ts
const next = t.state.content[0].text + chunk;
t.state.content[0].text = next.length > CAP ? next.slice(next.length - CAP) : next;
t.state.truncation.totalBytes += chunk.length; // <- the other path
```
### Evidence
50 KB window, 200-byte chunks, no flush in between:
| held-back writes | ops | bytes |
| --- | --- | --- |
| 1 | 3 | 0.3 KB |
| 100 | 201 | 26 KB |
| **1000** | **2001** | **264 KB** |
Without interleaving the same workload gives 1 op and 51 KB. The window is
bounded at 50 KB; the ops describing it are not.
### Why it matters beyond memory
Any design where the sink **holds back** — rate limiting, backpressure, a slow
consumer — depends on held-back writes costing nothing. They currently cost
everything. See `../../04-tool-output/rate-limiting.md`.
### Candidate fix
Fold at **record** time, keyed by path rather than by adjacency: one slot per
path holding at most two ops (`[s]`, `[d]`, `[t]`, `[a]`, or `[t, a]`), emitted in
first-touch order.
A prototype reached **2 ops / 51 KB** on the same workload. It is not included
here because it was not verified to the standard the rest of this unit meets.
**Four folding rules, each a specific claim about the vocabulary:**
1. A later `s` supersedes an earlier `s` on the same path — but **not** an earlier
`d`, which must survive or the key changes position on reinsertion (§5.1).
2. Consecutive `a` merge. Once the merged append is **strictly longer** than the
value, the window has been sliding and the append is the more expensive
spelling, so collapse to `s`. Strictly: while a string is still *growing*
toward its cap the append equals the value, and collapsing there would resend
the whole string every batch.
3. `t` and `a` on one path **commute** — the truncate drops from the front, the
append adds to the end — so a rolling window's `t,a,t,a,...` folds to one of
each. Without this, pairwise merging never fires at all.
4. `t` or `a` following a pending `s` changes nothing: the `s` already means "this
path becomes its current value".
**The trap that prototype hit.** Slots emit in first-touch order, which loses the
fact that a child write happened *before* a later parent write:
```
script : a={x:1}; a.b=99; a={c:2}
producer: {"a":{"c":2}}
replica : {"a":{"c":2,"b":99}} <- WRONG
```
A set or delete must invalidate every pending slot **below** its path, at record
time. The existing `verify-dead` style test passed 1190 sequences while this was
broken, because its generator only nested one level. Any test for this must
generate genuinely nested paths with parent overwrites interleaved between child
writes.
---
## D2. `overlap()` was 94.5% of prototype tracker time — closed
**Historical severity: high. Production decision:** do not add an explicit append/truncate API. The prototype profile was dominated by a slow `startsWith` path; production uses a flattened-slice comparison and measured 17.8–18.7 µs per 200 KB assistant append flush and 2.43–2.46 µs per 50 KB rolling-window flush locally. See [`append-decision.md`](append-decision.md). The original evidence follows.
### Problem
The tracker infers "you appended and evicted" by searching two 50 KB strings.
Each new value produced by `next.slice(-CAP)` is a V8 `SlicedString` — a lazy
substring — and `indexOf`/`endsWith` force it to **flatten**, copying 50 KB per
call.
### Evidence
CPU profile of a 20 000-write rolling-window loop, node, flush at the end:
```
94.5% overlap
2.7% (anon)
1.2% (garbage collector)
0.5% set
0.3% emit
0.0% fold
```
And the flattening is directly observable:
| `overlap()` on a 50 KB window | µs |
| --- | --- |
| the same two strings reused | 37 |
| **a fresh sliced string each call** | **149** |
An earlier benchmark reported 29 µs because it reused two already-flattened
strings 2000 times — a case that never occurs in the real loop. **Do not benchmark
string operations against a reused fixture.**
### Candidate fix
Do not infer what the producer already knows. `ToolOutput` performs the windowing,
so it can emit the ops directly rather than assigning a new string and having the
tracker search for the difference:
```ts
out.append(chunk); // -> ["t", path, dropped], ["a", path, chunk]
```
Two integers — chars appended, chars evicted — describe everything that happened
between any two observations of a bounded window. A prototype of this shape held
**exactly the window** in memory under every load, with no ops array, no overlap
detection and no folding rules at all.
This does **not** remove `overlap`: it stays for the general case where a producer
assigns a whole new string and the delta genuinely has to be discovered. It
removes it from the hot path.
### Do not
Do not "optimise" `overlap` itself before doing the above. Its algorithm is
already correct and bounded (`maxOverlapScan`); the cost is string flattening,
which no probe strategy avoids.
---
## D3. Nested-path proxy overhead is next, and currently invisible
**Severity: unknown — measure before acting.**
`overlap` dominates so completely that nothing else shows in a profile. But two
benchmarks disagree in a way that points at path depth:
| | µs per write |
| --- | --- |
| top-level path (`{ text }`) | 63 |
| nested path (`{ content: [{ text }] }`) | 244 |
4x for depth alone. Each nested property access wraps a child proxy and builds a
cache key by joining the path, so a read of `content[0].text` allocates on every
access.
**Re-profile only if production delta cost becomes material.** This may be the next bottleneck or it may be
nothing; the current numbers cannot distinguish them because `overlap` swamps both.
---
## D4. Things measured and found NOT worth doing
Recorded so nobody repeats the work. All were under **1.5% combined** in the
profile above.
**A path trie replacing `JSON.stringify(path)` keys.** Genuinely faster in
isolation — 6x on the `LaneSnapshot` shape, 93x at 500 distinct paths — because
stringify cost scales with path depth while a trie walk is one `Map.get` per
segment. But the whole slot layer is 0.5% of runtime. Build it only if a current production profile says so.
**Trie as storage vs trie as index.** If a trie is ever built, it should map path
to a **position in the ops array**, not hold the ops. Holding them loses first-touch
ordering and forces a sort at flush to rebuild it. Measured difference: none
(1.4x / 0.9x / 1.0x). The argument is structural, not performance.
**A `maxDepth` heuristic to skip descendant scanning.** Rejected: the flat map was
losing on the common case too, so the heuristic would have hidden a real cost
rather than removed it.
**Eager string concatenation in the `a`+`a` merge.** Real (it rebuilds a growing
string per write) but worth ~5%. Compare lengths and materialise only when
emitting.
---
## D5. Measurement errors made while producing these numbers
Listed because each one produced a confident wrong conclusion, and the same traps
are still there.
- **Benchmarking through `tsx`** — inflated everything 2.6x.
- **Reusing a flattened string** in a string benchmark — hid the real cost 4x.
- **Repetitive fixtures** (`"x".repeat(n)`) — overlap detection hits its candidate
bound and gives up, so the benchmark measures the fallback path, not the real one.
Use varied text.
- **Measuring a cache hit as if it were work** — `Markdown.render()` returns
`cachedLines` when text and width are unchanged, so a repeat-the-same-call loop
measured 0.065 ms for something that costs 0.86 ms.
- **Reasoning about complexity while the constant dominated** — twice.
- **Measuring heap instead of retained size** — a fixture generating 800 x 1 MB
strings reported 40 MB of "growth" that was its own garbage.

39
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/append-decision.md vendored Normal file
View File

@@ -0,0 +1,39 @@
# Decision: no explicit text append/truncate API
**Status: closed.** Do not add `appendText`, explicit truncate, a text-specific dirty-node kind, or proxy-to-tracker lookup machinery.
## Why it was considered
The tracker marks dirty paths and compares them with its accepted baseline at `flush()`. For a growing or rolling string, confirming the relationship can scan the retained string even though the producer already knows it appended or evicted text.
The original benchmark made this look expensive because its append fast path used `after.startsWith(before)`. V8 walked the producer's cons string character by character. Production Chord now uses:
```ts
if (after.length > before.length && after.slice(0, before.length) === before) {
// emit append
}
```
The slice flattens once and comparison uses the native string path. Flush-time dirty tracking also removed the prototype's retained-op amplification.
## Local confirmation
Measured 2026-09-01 against `origin/dev` at `1a7bc80e7`, using `packages/chord/src/delta/index.ts` directly under Node 26.0.0 on an Apple M5 Max. Each workload used 3,000 warmups followed by 11 samples of 10,000 mutation-plus-flush iterations; a second process reproduced the result.
| Workload | Median µs/flush, run 1 | Median µs/flush, run 2 |
| --- | ---: | ---: |
| 200 KB assistant string, append 8 characters | 18.68 | 17.81 |
| 50 KB rolling window, slide 32 varied characters | 2.46 | 2.43 |
| Transcript push, one small entry | 0.74 | 0.78 |
The assistant tracker started with 200,000 varied characters, appended `" abcdef"`, and flushed each append. The rolling tracker started with 50,000 varied characters, then assigned `text.slice(32) + chunk` using a distinct 32-character `chunk:<base36 index>:durable-stream` value on every flush; every flush was asserted to emit `t` + `a`. The transcript tracker pushed `{ id: "e<index>", text: "message <index>" }` and asserted one `p` per flush. Setup and garbage collection were outside each timed loop.
At 100 assistant updates per second, the growing-string case consumes about 1.8 ms/s, approximately 0.18% of one core. The measured rolling-window cost is itself too small to justify the abandoned explicit API.
## Why the API is rejected
The prototype required a text-specific dirty-node state, proxy-to-tracker lookup, repeated-append/drop folding, and interaction rules for later whole-value replacement. It produced subtle silent failures: one implementation fell through to the ordinary differ while tests still passed, and another drifted when drops reached into earlier appends.
That complexity does not justify saving microseconds below the surrounding replication, isolation, and rendering costs. Keep ordinary string mutation and the generic Chord op vocabulary.
Reopen only when a production profile shows delta flush time is a meaningful fraction of a real workload. Re-measure the generic fast path first; a regression there is cheaper and safer to fix than adding a producer-specific API.

826
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta-impl.ts vendored Normal file
View File

@@ -0,0 +1,826 @@
// ─────────────────────────────────────────────────────────────────────────────
// harness/delta — intent-recording change tracking over plain JSON.
//
// Depends on nothing else in the harness. Session storage, the runtime and the
// facet host consume it; keep the arrows pointing that way.
// ─────────────────────────────────────────────────────────────────────────────
export type JsonValue = null | boolean | number | string | JsonValue[] | { [k: string]: JsonValue };
export type Seg = string | number;
export type Path = readonly Seg[];
export type NonEmptyPath = readonly [Seg, ...Seg[]];
/** A path inline, or an id assigned by the encoder on second use. */
export type PathRef<P extends Path = Path> = P | number;
/**
* Tuples are the form — in memory, on the wire, on disk.
*
* `r` is the ONLY op that replaces a whole value. `s`/`d`/`a`/`t` cannot target
* the root: the type forbids it. `p` may, and only because a tracked value can
* itself be an array — but a `p` that replaces its entire target is normalised to
* `r`/`s` at record time, so a root `p` is always a partial modification.
*
* `Op` knows nothing about the path dictionary. Interning, id references and
* omitted paths live in `WireOp` and exist only between `encode` and `decode`.
*/
export type Op =
| readonly ["r", JsonValue]
| readonly ["s", NonEmptyPath, JsonValue]
| readonly ["d", NonEmptyPath]
| readonly ["a", NonEmptyPath, string]
| readonly ["t", NonEmptyPath, number]
| readonly ["p", Path, number, number, JsonValue[]];
/**
* What crosses a boundary. Adds two compressions and nothing else:
*
* ["#", id, path] defines an id, emitted on a path's SECOND use
* a numeric PathRef references a previously defined id
* a shortened tuple reuses the previous op's path; arity disambiguates
*
* ["r", value] carries no path, so it encodes to itself — which is why isBase
* works unchanged on either vocabulary.
*/
export type WireOp =
| readonly ["r", JsonValue]
| readonly ["s", PathRef<NonEmptyPath>, JsonValue] | readonly ["s", JsonValue]
| readonly ["d", PathRef<NonEmptyPath>] | readonly ["d"]
| readonly ["a", PathRef<NonEmptyPath>, string] | readonly ["a", string]
| readonly ["t", PathRef<NonEmptyPath>, number] | readonly ["t", number]
| readonly ["p", PathRef, number, number, JsonValue[]]
| readonly ["p", number, number, JsonValue[]]
| readonly ["#", number, Path];
// ─── Classification ──────────────────────────────────────────────────────────
export const isReplace = (op: Op | WireOp): boolean => op[0] === "r";
/**
* A batch begins with a replacement. Flush guarantees `r` is at index 0 or absent,
* so this is exact rather than a heuristic.
*/
export const isBase = (ops: readonly (Op | WireOp)[]): boolean =>
ops.length > 0 && ops[0]![0] === "r";
// ─── Overlap ─────────────────────────────────────────────────────────────────
/**
* Longest suffix of `a` that is a prefix of `b`. Probes with indexOf and verifies
* with endsWith, so the hot loops are native. A hand-written KMP is
* asymptotically equivalent and ~47x slower in practice.
*
* Always correct: the returned n satisfies a.endsWith(b.slice(0, n)).
*/
export function overlap(a: string, b: string, scan: number, probe = 64, maxCandidates = 8): number {
if (a.length === 0 || b.length === 0 || scan === 0) return 0;
const tail = a.length > scan ? a.slice(a.length - scan) : a;
// A probe of length h can only find overlaps of at least h — the head must
// actually occur in `a`. So try a long head first (few candidates, and it
// catches the large overlaps a rolling window produces), then fall back to one
// character, which finds any overlap at the cost of more candidates.
//
// Candidates are bounded because repetitive output — a build log, or any run of
// one character — makes a long head match at thousands of positions, each
// costing a full endsWith. Giving up returns 0, which emits a set: larger, never
// wrong. Unbounded, a 50 KB window of one repeated character took 4.7 s per
// 2000 slides against 90 ms bounded.
for (const h of [Math.min(probe, b.length), 1]) {
const head = b.slice(0, h);
let tried = 0;
for (let k = tail.indexOf(head); k !== -1; k = tail.indexOf(head, k + 1)) {
if (++tried > maxCandidates) break;
const n = tail.length - k;
if (n <= b.length && tail.endsWith(b.slice(0, n))) return n;
}
if (h === 1) break;
}
return 0;
}
// ─── Tracker ─────────────────────────────────────────────────────────────────
export interface TrackerOptions {
maxOverlapScan?: number;
coalesce?: boolean;
}
export interface Tracker<T extends object> {
/**
* Mutate this. Plain TS: assignment, push, splice, delete, all of it.
*
* Assigning to it replaces the whole value: emits `["r", next]` and discards
* ops recorded before it, which describe a value that no longer exists.
*
* `tracker.state = tracker.target` is therefore a checkpoint — it forces a base
* batch without changing anything. Discarding the pending ops is correct
* because the proxy mutates the target directly, so the value already carries
* them.
*/
state: T;
/** The untracked backing object. Reads only — writing here emits nothing. */
readonly target: T;
flush(): Op[];
/**
* Clear pending ops and make the next flush a base batch, without changing
* the value.
*
* Recovery replays from the last base batch, so a producer that wants "at most
* N batches to replay" calls this every N. The pending ops are already
* reflected in the value — the proxy mutates the target directly — so
* discarding them loses nothing.
*/
rebase(): void;
/** Discard without emitting. */
discard(): void;
readonly dirty: boolean;
}
const isObj = (v: unknown): v is object => v !== null && typeof v === "object";
const clone = <T>(v: T): T => (isObj(v) ? (structuredClone(v) as T) : v);
const INDEX = /^(?:0|[1-9]\d*)$/;
const norm = (t: object, k: string | symbol): Seg | symbol =>
typeof k === "symbol" ? k : Array.isArray(t) && INDEX.test(k) ? Number(k) : k;
const MUTATORS = new Set(["push", "pop", "shift", "unshift", "splice", "sort", "reverse", "fill", "copyWithin"]);
export function track<T extends object>(root: T, options: TrackerOptions = {}): Tracker<T> {
const scan = options.maxOverlapScan ?? 65_536;
const doCoalesce = options.coalesce ?? true;
let ops: Op[] = [];
const emit = (op: Op) => {
ops.push(op);
};
/**
* Reject reserved keys at record time, not just on apply.
*
* `JSON.parse('{"__proto__":{}}')` yields an OWN property, so a producer can
* hold state containing one without doing anything unusual. The tracker would
* then emit a path the applier refuses — state that cannot be replicated. Fail
* where the mistake is, at the write.
*/
const guard = (seg: Seg | symbol): Seg => {
if (typeof seg === "symbol") throw new UnsafePathError(String(seg));
if (typeof seg === "string" && FORBIDDEN.has(seg)) throw new UnsafePathError(seg);
return seg;
};
/**
* A splice covering the whole target is a replacement written the long way.
* Only the interception knows the pre-splice length, so this is a record-time
* rewrite, not something flush can recover.
*/
const spliceOrReplace = (path: Path, before: number, index: number, remove: number, items: JsonValue[]) => {
if (index === 0 && remove === before) {
if (path.length === 0) emit(["r", items]);
else emit(["s", path as NonEmptyPath, items]);
return;
}
emit(["p", [...path], index, remove, items]);
};
const cache = new WeakMap<object, Map<string, object>>();
const wrap = <V extends object>(obj: V, path: Path): V => {
let byPath = cache.get(obj);
if (byPath === undefined) cache.set(obj, (byPath = new Map()));
const cacheKey = path.join("\u0000");
const hit = byPath.get(cacheKey);
if (hit !== undefined) return hit as V;
const proxy = new Proxy(obj, {
get(t, k, r) {
if (Array.isArray(t) && typeof k === "string" && MUTATORS.has(k)) {
return (...args: unknown[]) => {
const before = t.length;
const result = (Array.prototype[k as "push"] as Function).apply(t, args);
const items = (xs: unknown[]) => xs.map(clone) as JsonValue[];
switch (k) {
case "push": spliceOrReplace(path, before, before, 0, items(args)); break;
case "unshift": spliceOrReplace(path, before, 0, 0, items(args)); break;
case "pop": if (before > 0) spliceOrReplace(path, before, before - 1, 1, []); break;
case "shift": if (before > 0) spliceOrReplace(path, before, 0, 1, []); break;
case "splice": {
const raw = Number(args[0] ?? 0);
const index = raw < 0 ? Math.max(0, before + raw) : Math.min(raw, before);
const remove = args.length < 2 ? before - index : Math.max(0, Math.min(Number(args[1]), before - index));
spliceOrReplace(path, before, index, remove, items(args.slice(2)));
break;
}
// sort/reverse/fill/copyWithin permute wholesale; intent is not a splice.
default:
if (path.length === 0) emit(["r", clone(t) as JsonValue]);
else emit(["s", path as NonEmptyPath, clone(t) as JsonValue]);
}
return result;
};
}
const v = Reflect.get(t, k, r);
return isObj(v) ? wrap(v, [...path, guard(norm(t, k))]) : v;
},
set(t, k, v, r) {
// `length` is a real mutation but not a document location.
if (Array.isArray(t) && k === "length") {
const before = t.length;
const next = Number(v);
if (next < before) spliceOrReplace(path, before, next, before - next, []);
else if (next > before) emit(["p", [...path], before, 0, Array(next - before).fill(null)]);
return Reflect.set(t, k, v, r);
}
const at = [...path, guard(norm(t, k))] as unknown as NonEmptyPath;
if (v === undefined) { // JSON has no undefined
emit(["d", at]);
return Reflect.deleteProperty(t, k);
}
const prev = (t as Record<string | symbol, unknown>)[k];
if (typeof prev === "string" && typeof v === "string") {
if (prev === v) return true;
if (v.length > prev.length && v.startsWith(prev)) {
emit(["a", at, v.slice(prev.length)]); // fast path, no probe
} else {
const ov = overlap(prev, v, scan);
if (ov === 0) emit(["s", at, v]);
else {
emit(["t", at, prev.length - ov]);
if (v.length > ov) emit(["a", at, v.slice(ov)]);
}
}
} else {
assertJsonValue(v);
emit(["s", at, clone(v) as JsonValue]);
}
return Reflect.set(t, k, v, r);
},
deleteProperty(t, k) {
emit(["d", [...path, guard(norm(t, k))] as unknown as NonEmptyPath]);
return Reflect.deleteProperty(t, k);
},
});
byPath.set(cacheKey, proxy);
return proxy as V;
};
let state = wrap(root, []);
// The first flush is always a base batch. A consumer starts with nothing, so
// a stream that opens with deltas has nothing to apply them to — and that
// fails at runtime, in the consumer, rather than where the mistake is.
let forceBase = true;
return {
get state() { return state; },
// A setter, so the obvious thing works. Without it `tracker.state = next`
// silently swaps the proxy for a plain object and stops tracking.
set state(next: T) {
ops = [];
root = next;
state = wrap(root, []);
forceBase = true;
},
get target() { return root; },
// Same effect as assigning the current value back, minus the re-wrap: the
// value has not changed, so the proxy cache is still valid.
rebase() { ops = []; forceBase = true; },
get dirty() { return ops.length > 0; },
discard() { ops = []; },
flush() {
if (forceBase) {
forceBase = false;
ops = [];
return [["r", clone(root) as JsonValue]];
}
if (ops.length === 0) return [];
let out = ops;
ops = [];
// Everything before a replacement is dead.
const lastReplace = out.findLastIndex(isReplace);
if (lastReplace > 0) out = out.slice(lastReplace);
if (doCoalesce) out = coalesce(dropDead(out));
return out;
},
};
}
const samePath = (a: Op, b: Op): boolean => {
const pa = pathOf(a);
const pb = pathOf(b);
if (pa === undefined || pb === undefined) return false;
return JSON.stringify(pa) === JSON.stringify(pb);
};
function pathOf(op: Op): Path | undefined {
return op[0] === "r" ? undefined : op[1];
}
/**
* Drop ops that a later op makes unreachable.
*
* Ops apply in order, so an op is dead if a later one overwrites the whole
* subtree it lives in. Only `s`, `d` and `r` do that — `a`, `t` and `p` modify
* what is already there, so they never dominate.
*
* O(n * d): walk backwards keeping a map of overwritten paths, and for each op
* test its own prefixes against that map. `d` is the path depth, which is small
* and bounded. The obvious formulation — compare each op against every
* dominator — is O(n^2 * d) and degrades on exactly the wide-flush input this
* pass exists to clean up. Measured: 12-14 microseconds per op, flat from 100 to
* 5000 ops, and 10001 redundant writes collapse to 2 emitted ops.
*
* OBJECT KEY ORDER IS NOT PRESERVED. A producer that does
* `set c; set a; delete c; set c` leaves `c` last, because deleting and
* reinserting moves it. The replica receives only the surviving `set c`, applies
* it to a base where `c` never existed, and `c` lands first. Values are
* identical — verified over 1823 randomised sequences with zero value
* mismatches — but the serialised forms can differ.
*
* This is fine because nothing compares a replica against its producer: the
* consumer holds the state and nobody diffs it back. The one visible case is a
* consumer that iterates keys for display, where two views could order rows
* differently. It requires a delete-and-recreate of the same key within one
* flush. Do not hash or content-address a replicated value.
*/
function dropDead(ops: Op[]): Op[] {
const overwritten = new Map<string, "s" | "d">();
const kept: Op[] = [];
for (let i = ops.length - 1; i >= 0; i--) {
const op = ops[i]!;
if (op[0] === "r") {
kept.push(op);
overwritten.clear();
overwritten.set("", "s"); // the root is now overwritten
continue;
}
const path = opPath(op);
if (path === undefined) { kept.push(op); continue; } // arity form, or an interned id
let dead = overwritten.has("");
let key = "";
for (let k = 0; k < path.length && !dead; k++) {
key += `\u0000${String(path[k])}`;
const kind = overwritten.get(key);
if (kind === undefined) continue;
// A `d` at exactly a later `s`'s path is NOT dead. Dropping it leaves the
// key in its original slot, while the producer's delete-then-set moved it
// to the end: same value, different serialisation.
if (k === path.length - 1 && op[0] === "d" && kind === "s") break;
dead = true;
}
if (dead) continue;
kept.push(op);
if (op[0] === "s" || op[0] === "d") overwritten.set(key, op[0]);
}
return kept.reverse();
}
/** Total, because `Op` paths are always inline. */
function opPath(op: Op): Path | undefined {
return op[0] === "r" ? undefined : op[1];
}
/** Merge adjacent ops describing the same path. Lossless. */
function coalesce(ops: Op[]): Op[] {
const out: Op[] = [];
for (const op of ops) {
const last = out[out.length - 1];
if (last !== undefined && samePath(last, op)) {
if (last[0] === "a" && op[0] === "a") {
out[out.length - 1] = ["a", last[1], `${last[2]}${op[2]}`];
continue;
}
if (last[0] === "s" && op[0] === "s") { // later write wins
out[out.length - 1] = op;
continue;
}
if (last[0] === "s" && op[0] === "a" && typeof last[2] === "string") {
out[out.length - 1] = ["s", last[1], `${last[2]}${op[2]}`];
continue;
}
}
out.push(op);
}
return out;
}
// ─── Path safety ─────────────────────────────────────────────────────────────
/**
* Segments that reach the prototype chain.
*
* `JSON.parse` is safe on its own — it makes `__proto__` an own property. What is
* not safe is `parent[key] = value`, which is exactly what an applier does, and
* paths are data: `["s", ["__proto__", "isAdmin"], true]` pollutes
* `Object.prototype` for the whole process.
*
* Ops arrive from a facet, a plugin compartment, or a tool whose details may echo
* model output, so none of it is trusted input.
*/
const FORBIDDEN = new Set(["__proto__", "constructor", "prototype"]);
export class UnsafePathError extends Error {
// Not a parameter property: Node's --experimental-strip-types rejects those,
// and these files are meant to run under it directly.
readonly segment: Seg;
constructor(segment: Seg) {
super(`unsafe path segment: ${String(segment)}`);
this.segment = segment;
this.name = "UnsafePathError";
}
}
/**
* Rejects anything JSON.stringify would not round-trip.
*
* structuredClone is NOT a JSON check: it clones a Map happily, and the op then
* carries `{}` while the producer keeps a real Map — producer and replica diverge
* silently, forever.
*/
export function assertJsonValue(value: unknown, seen = new Set<unknown>()): void {
if (value === null) return;
const t = typeof value;
if (t === "string" || t === "boolean") return;
if (t === "number") {
if (!Number.isFinite(value as number)) throw new TypeError("non-finite number is not JSON");
return;
}
if (t !== "object") throw new TypeError(`${t} is not a JsonValue`);
if (seen.has(value)) throw new TypeError("cyclic value is not JSON");
seen.add(value);
if (Array.isArray(value)) {
for (const v of value) assertJsonValue(v, seen);
} else {
const proto = Object.getPrototypeOf(value);
if (proto !== Object.prototype && proto !== null) {
throw new TypeError(`${(value as object).constructor?.name ?? "object"} is not a JsonValue`);
}
if (Object.getOwnPropertySymbols(value).length > 0) {
throw new TypeError("symbol keys are not JSON");
}
for (const v of Object.values(value as object)) assertJsonValue(v, seen);
}
seen.delete(value);
}
/**
* Verb, arity and payload shape for a **decoded** op: paths inline, no `#`, no
* short forms. `apply` uses this.
*
* Validating `Op` against the wire grammar would be laxer than the type: a
* two-element `["s", value]` would pass, and `apply` would then read the value as
* a path. Each vocabulary gets the validator that matches it.
*/
export function assertValidOp(op: unknown): void {
if (!Array.isArray(op) || op.length === 0) throw new TypeError("op is not a tuple");
switch (op[0]) {
case "r": if (op.length !== 2) throw new TypeError("r arity"); assertJsonValue(op[1]); return;
case "s": if (op.length !== 3) throw new TypeError("s arity"); assertPathArg(op[1]); assertJsonValue(op[2]); return;
case "d": if (op.length !== 2) throw new TypeError("d arity"); assertPathArg(op[1]); return;
case "a": if (op.length !== 3 || typeof op[2] !== "string") throw new TypeError("a shape"); assertPathArg(op[1]); return;
case "t": if (op.length !== 3 || !Number.isInteger(op[2])) throw new TypeError("t shape"); assertPathArg(op[1]); return;
case "p": {
if (op.length !== 5) throw new TypeError("p arity");
assertPathArg(op[1]);
if (!Number.isInteger(op[2]) || op[2] < 0) throw new TypeError("p index");
if (!Number.isInteger(op[3]) || op[3] < 0) throw new TypeError("p remove");
if (!Array.isArray(op[4])) throw new TypeError("p items");
for (const v of op[4]) assertJsonValue(v);
return;
}
// Silently skipping an unknown verb is how a newer producer's op vanishes.
default: throw new TypeError(`unknown op verb: ${String(op[0])}`);
}
}
function assertPathArg(p: unknown): void {
if (!Array.isArray(p)) throw new TypeError("path is not an array");
assertSafePath(p as Path);
}
/** The same, for the wire grammar: ids and short forms are legal here. */
export function assertValidWireOp(op: unknown): void {
if (!Array.isArray(op) || op.length === 0) throw new TypeError("op is not a tuple");
const [verb] = op as unknown[];
const okRef = (r: unknown): void => {
if (typeof r === "number") {
if (!Number.isInteger(r) || r < 0) throw new TypeError("bad path id");
return;
}
// A string is not a path. Unchecked, `"a".slice(0, -1)` is `""`, so it
// resolves to the ROOT and writes there — a path that is not a path, accepted.
if (!Array.isArray(r)) throw new TypeError("path is not an array");
assertSafePath(r as Path);
};
switch (verb) {
case "r": if (op.length !== 2) throw new TypeError("r arity"); assertJsonValue(op[1]); return;
case "s": if (op.length === 3) { okRef(op[1]); assertJsonValue(op[2]); }
else if (op.length === 2) assertJsonValue(op[1]);
else throw new TypeError("s arity"); return;
case "d": if (op.length === 2) okRef(op[1]); else if (op.length !== 1) throw new TypeError("d arity"); return;
case "a": if (op.length === 3) { okRef(op[1]); if (typeof op[2] !== "string") throw new TypeError("a value"); }
else if (op.length === 2) { if (typeof op[1] !== "string") throw new TypeError("a value"); }
else throw new TypeError("a arity"); return;
case "t": if (op.length === 3) { okRef(op[1]); if (!Number.isInteger(op[2])) throw new TypeError("t count"); }
else if (op.length === 2) { if (!Number.isInteger(op[1])) throw new TypeError("t count"); }
else throw new TypeError("t arity"); return;
case "p": {
const [i, r, items] = op.length === 5 ? [op[2], op[3], op[4]] : op.length === 4 ? [op[1], op[2], op[3]] : [];
if (items === undefined) throw new TypeError("p arity");
if (op.length === 5) okRef(op[1]);
if (!Number.isInteger(i) || (i as number) < 0) throw new TypeError("p index");
if (!Number.isInteger(r) || (r as number) < 0) throw new TypeError("p remove");
if (!Array.isArray(items)) throw new TypeError("p items");
for (const v of items) assertJsonValue(v);
return;
}
case "#": {
if (op.length !== 3 || !Number.isInteger(op[1]) || !Array.isArray(op[2])) throw new TypeError("# shape");
assertSafePath(op[2] as Path);
return;
}
// Silently skipping an unknown verb is how a newer producer's op vanishes.
default: throw new TypeError(`unknown op verb: ${String(verb)}`);
}
}
export function assertSafePath(path: Path): void {
for (const seg of path) {
if (typeof seg === "string") {
if (FORBIDDEN.has(seg)) throw new UnsafePathError(seg);
} else if (!Number.isInteger(seg) || seg < 0) {
throw new UnsafePathError(seg);
}
}
}
/**
* An index may address an existing element or append exactly one past the end.
*
* This is not an arbitrary cap — it is what keeps the value a `JsonValue`. A
* sparse array does not survive a JSON round trip: holes serialise to `null` and
* return as real properties, so `arr[7] = x` on a length-3 array already produces
* state a replica cannot match. Rejecting the write is more honest than silently
* diverging.
*
* It also removes the denial of service it would otherwise permit:
* `["s", ["xs", 4294967290], 1]` allocates a 4.29-billion-entry array from one op.
* Growth stays possible and stays proportional — the tracker already emits
* `arr.length = n` as a splice of explicit nulls, whose op size grows with the
* gap, so a large growth costs a large op rather than a small one.
*/
function assertIndexInRange(parent: readonly unknown[], index: number): void {
if (index > parent.length) throw new UnsafePathError(index);
}
// ─── Applier ─────────────────────────────────────────────────────────────────
export class PathError extends Error {
readonly path: Path | number;
constructor(path: Path | number) {
super(`unresolvable path: ${JSON.stringify(path)}`);
this.path = path;
this.name = "PathError";
}
}
/**
* Apply ops to a plain mutable value. Returns the value, because `r` replaces it
* outright and cannot be done in place.
*
* Takes decoded ops. Path ids and omitted paths are a wire concern — run
* `decode` first if the ops came from a boundary.
*/
export function apply<T>(target: T, ops: readonly Op[]): T {
let root = target as unknown as JsonValue;
for (const op of ops) {
assertValidOp(op);
if (op[0] === "r") {
// Adopted, not copied. The consumer owns the batch it was handed.
//
// Fanning one batch out to several consumers in-process therefore makes
// their replicas alias each other. That is an ownership rule, not a
// defect: copy the batch at the fan-out point, or let each consumer
// decode its own. A batch that crosses a real boundary is already
// distinct, because serialisation produces fresh objects.
root = op[1];
continue;
}
const path = op[1];
assertSafePath(path);
if (op[0] === "p") {
const target_ = path.length === 0 ? root : resolve(root, path);
if (!Array.isArray(target_)) throw new PathError(path);
target_.splice(op[2], op[3], ...op[4]);
continue;
}
// s/d/a/t can never target the root — the type forbids it.
const parent = resolve(root, path.slice(0, -1)) as Record<Seg, JsonValue>;
const key = path[path.length - 1]!;
if (Array.isArray(parent) && typeof key === "number") assertIndexInRange(parent, key);
// defineProperty rather than assignment: a setter inherited from the prototype
// chain would otherwise run on write.
const write = (value: JsonValue) => {
Object.defineProperty(parent, key, { value, writable: true, enumerable: true, configurable: true });
};
const read = (): unknown => (Object.hasOwn(parent, key) ? parent[key] : undefined);
switch (op[0]) {
case "s": write(op[2]); break;
case "d": Array.isArray(parent) ? (parent as unknown as JsonValue[]).splice(Number(key), 1) : delete parent[key]; break;
case "a": write(`${String(read() ?? "")}${op[2]}`); break;
case "t": write(String(read() ?? "").slice(op[2])); break;
}
}
return root as unknown as T;
}
function resolve(root: JsonValue, path: Path): JsonValue {
let node: JsonValue = root;
for (const seg of path) {
if (!isObj(node)) throw new PathError(path);
// Own properties only: an inherited getter must not run, and a walk must not
// escape the value into the prototype chain.
if (!Object.hasOwn(node, seg as PropertyKey)) throw new PathError(path);
node = (node as Record<Seg, JsonValue>)[seg]!;
}
if (!isObj(node)) throw new PathError(path);
return node;
}
// ─── Codec ───────────────────────────────────────────────────────────────────
//
// Path interning and arity omission, per `delta.md` §4. These live between the
// tracker and a boundary; `Op` and `apply` know nothing about them.
//
// ONE PAIR PER STREAM. The table spans an entire subscription or file: a path
// interned in batch 3 is referenced in batch 40. A second consumer that
// subscribes at batch 40 has never seen the definition, so it needs its own
// encoder — sharing one across consumers hands the late subscriber ids it cannot
// resolve. A base batch does not rescue it, because `["r", value]` carries no
// path refs and leaves the table empty.
const pathKey = (path: Path): string => path.join("\u0000");
export interface Encoder {
encode(ops: readonly Op[]): WireOp[];
}
/**
* Intern on SECOND use. A definition costs more than the path it replaces, so
* interning on first use loses on the many paths written exactly once —
* measured worse than no interning at all (`delta.md` §4.1).
*/
export function encoder(): Encoder {
const seen = new Set<string>();
const ids = new Map<string, number>();
let nextId = 0;
let previous: string | undefined; // last path in THIS batch
return {
encode(ops) {
// Arity omission is scoped to a batch. Letting it span batches would make
// a batch's first op depend on the previous batch's last one, so a reader
// that skips or reorders a batch decodes into the wrong path. Ids are the
// only cross-batch state, and the dictionary makes those explicit.
previous = undefined;
const out: WireOp[] = [];
for (const op of ops) {
if (op[0] === "r") {
out.push(op);
// A base batch is a RECOVERY POINT: a reader replays from the last one
// with a fresh decoder. So everything after it must be self-contained.
// Keeping ids across a replacement emits references to definitions the
// reader never saw — recovery fails with an unresolvable path id.
seen.clear();
ids.clear();
nextId = 0;
previous = undefined;
continue;
}
const path = op[1];
const key = pathKey(path);
// Same path as the previous op: drop the ref entirely.
if (key === previous) {
switch (op[0]) {
case "s": out.push(["s", op[2]]); break;
case "d": out.push(["d"]); break;
case "a": out.push(["a", op[2]]); break;
case "t": out.push(["t", op[2]]); break;
case "p": out.push(["p", op[2], op[3], op[4]]); break;
}
continue;
}
let ref: PathRef = path;
const existing = ids.get(key);
if (existing !== undefined) {
ref = existing;
} else if (seen.has(key)) {
const id = nextId++;
ids.set(key, id);
out.push(["#", id, path]); // second use: define, then reference
ref = id;
} else {
seen.add(key); // first use: inline
}
switch (op[0]) {
case "s": out.push(["s", ref as PathRef<NonEmptyPath>, op[2]]); break;
case "d": out.push(["d", ref as PathRef<NonEmptyPath>]); break;
case "a": out.push(["a", ref as PathRef<NonEmptyPath>, op[2]]); break;
case "t": out.push(["t", ref as PathRef<NonEmptyPath>, op[2]]); break;
case "p": out.push(["p", ref, op[2], op[3], op[4]]); break;
}
previous = key;
}
return out;
},
};
}
export interface Decoder {
decode(wire: readonly WireOp[]): Op[];
}
export function decoder(): Decoder {
const paths = new Map<number, Path>();
return {
decode(wire) {
let previous: Path | undefined; // scoped to the batch, as in encode
const out: Op[] = [];
for (const op of wire) {
assertValidWireOp(op);
if (op[0] === "#") {
assertSafePath(op[2]);
paths.set(op[1], op[2]);
continue;
}
if (op[0] === "r") {
out.push(op);
previous = undefined;
continue;
}
// Arity tells us whether a ref is present: the short forms omit it.
const short =
(op[0] === "d" && op.length === 1) ||
(op[0] !== "d" && op[0] !== "p" && op.length === 2) ||
(op[0] === "p" && op.length === 4);
let path: Path;
if (short) {
if (previous === undefined) throw new PathError([]);
path = previous;
} else {
const ref = op[1] as PathRef;
if (typeof ref === "number") {
const resolved = paths.get(ref);
if (resolved === undefined) throw new PathError(ref);
path = resolved;
} else {
path = ref;
}
previous = path;
}
switch (op[0]) {
case "s": out.push(["s", path as NonEmptyPath, (short ? op[1] : op[2]) as JsonValue]); break;
case "d": out.push(["d", path as NonEmptyPath]); break;
case "a": out.push(["a", path as NonEmptyPath, (short ? op[1] : op[2]) as string]); break;
case "t": out.push(["t", path as NonEmptyPath, (short ? op[1] : op[2]) as number]); break;
case "p": {
const [i, r, items] = short
? [op[1] as number, op[2] as number, op[3] as JsonValue[]]
: [op[2] as number, op[3] as number, op[4] as JsonValue[]];
out.push(["p", path, i, r, items]);
break;
}
}
}
return out;
},
};
}

113
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.bench.ts vendored Normal file
View File

@@ -0,0 +1,113 @@
/**
* Not a test — a guard against the cliffs found during implementation. Reports
* units you can reason about: cost per WRITE, and bytes on the wire relative to
* bytes consumed.
*
* npx vitest bench delta.bench.ts
*
* The headline finding: cost is per write, not per byte. A 100-byte chunk costs
* about the same as a 1 KB chunk, because the fixed cost is the window slice and
* the overlap probe. So chunk size — set by the exec env's coalescing interval,
* not by this package — decides both throughput and wire size.
*/
import { bench, describe } from "vitest";
import { decoder, encoder, track } from "./delta-impl.ts";
const WORDS = "the quick brown fox jumps over lazy dog while considering strategies".split(" ");
const text = (n: number): string => {
let s = "";
while (s.length < n) s += `${WORDS[Math.floor(Math.random() * WORDS.length)]} `;
return s.slice(0, n);
};
const CAP = 50_000;
const WRITES = 500;
const FLUSH_EVERY = 20;
/** One write = one chunk handed to the sink. Returns nothing; bench times it. */
function drive(initial: string, chunkSize: number, cap = CAP): void {
const t = track({ out: initial });
for (let i = 0; i < WRITES; i++) {
const next = t.state.out + text(chunkSize);
t.state.out = next.length > cap ? next.slice(next.length - cap) : next;
if (i % FLUSH_EVERY === FLUSH_EVERY - 1) t.flush();
}
t.flush();
}
/**
* Measured on the reference machine, 500 writes each. Reproduced here so a
* regression is legible rather than just "slower".
*
* append only, 100 B 127 µs/write 8k writes/s 101% of input on wire
* rolling, 100 B 176 µs/write 6k writes/s 135% of input on wire
* rolling, 1 KB 68 µs/write 15k writes/s 104% of input on wire
* rolling, 8 KB 366 µs/write 3k writes/s 31% of input on wire
* pathological, 100 B 307 µs/write 3k writes/s 727% of input on wire
*
* Two things to notice.
*
* 135% at 100-byte chunks: past the cap every write emits `truncate` + `append`,
* and two op envelopes exceed a 100-byte payload — we put MORE on the wire than we
* consumed. At 8 KB it is 31%. Flush granularity is therefore a size lever, not
* just an update-rate lever, and the exec env's coalescing is what keeps chunks
* off the bad end of that curve.
*
* The pathological row is bounded only because the overlap candidate scan is
* capped. Unbounded it was 4.7 s for 2000 writes against 93 ms; the cap makes it
* give up and emit a set — larger, never wrong.
*/
describe("rolling window, 50 KB cap", () => {
bench("100 B chunks", () => { drive(text(CAP), 100); });
bench("1 KB chunks", () => { drive(text(CAP), 1024); });
bench("8 KB chunks", () => { drive(text(CAP), 8192); });
bench("pathological: one repeated character", () => { drive("x".repeat(CAP), 100); });
});
describe("below the cap", () => {
// The dominant case for most tools: startsWith fast path, no probe at all,
// and one append per flush rather than a truncate/append pair.
bench("append only, never fills", () => { drive("", 100, Number.MAX_SAFE_INTEGER); });
});
// ── Codec: what path interning and arity omission actually save ────────────
//
// Reported as wire bytes relative to inline paths, on three shapes:
// - one hot path (a rolling tool-output window): omission does the work
// - a few alternating paths (lane state): interning does the work
// - many distinct paths: neither helps, and definitions cost a little
describe("codec", () => {
const streams = {
"one hot path": () => {
const t = track({ content: [{ type: "text", text: "" }] });
return Array.from({ length: 200 }, (_, i) => {
(t.state.content[0] as { text: string }).text += `line ${i} of build output\n`;
return t.flush();
});
},
"four alternating paths": () => {
const t = track({ a: "", b: "", c: "", d: "" });
return Array.from({ length: 200 }, (_, i) => {
for (const k of ["a", "b", "c", "d"] as const) t.state[k] += `${i}`;
return t.flush();
});
},
"200 distinct paths": () => {
const root: Record<string, number> = {};
for (let i = 0; i < 200; i++) root[`f${i}`] = 0;
const t = track(root);
return Array.from({ length: 5 }, (_, r) => {
for (let i = 0; i < 200; i++) t.state[`f${i}`] = r + 1;
return t.flush();
});
},
};
for (const [label, build] of Object.entries(streams)) {
bench(label, () => {
const enc = encoder();
const dec = decoder();
for (const ops of build()) dec.decode(enc.encode(ops));
});
}
});

133
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.examples.ts vendored Normal file
View File

@@ -0,0 +1,133 @@
/**
* Runnable illustrations of the two things that are easy to get wrong.
*
* node --experimental-strip-types delta.examples.ts
*
* Nothing here is a test — `delta.test.ts` covers the same ground with
* assertions. This exists to be read.
*/
import { apply, decoder, encoder, isBase, track, type Op } from "./delta-impl.ts";
const show = (label: string, ops: Op[]) => {
console.log(` ${label.padEnd(22)} ${ops.length} op(s), base=${isBase(ops)}`);
for (const op of ops) console.log(` ${trunc(JSON.stringify(op), 66)}`);
};
const trunc = (s: string, n: number) => (s.length <= n ? s : `${s.slice(0, n)}…`);
// ─────────────────────────────────────────────────────────────────────────
// 1. Replacing the whole value
// ─────────────────────────────────────────────────────────────────────────
console.log("\n1. REPLACING THE WHOLE VALUE\n");
const before = { user: { id: "u1", name: "ada" }, items: ["a", "b"], note: "n".repeat(120) };
const after = { user: { id: "u2", name: "bob" }, items: ["x"], note: "m".repeat(120) };
{
// The wrong way. `state` is a property of the TRACKER, not of the tracked
// object. Without a setter this swaps the proxy for a plain object and every
// later mutation is silently untracked — no error, no ops.
//
// The implementation defines `state` as a setter precisely so this works.
const t = track<Record<string, unknown>>(structuredClone(before));
t.flush(); // drain the opening base batch
t.state.items = ["changed first"]; // recorded…
t.state = structuredClone(after); // …then discarded: it described a dead value
show("assign to .state", t.flush());
t.state.user = { id: "u3" }; // still tracked afterwards
show(" and still tracked", t.flush());
}
{
// Dead-op elimination: a field written three times emits one op, and a parent
// replaced after its child emits one. Object key order is not preserved by
// this pass — see delta.md §5.1.
const t = track<Record<string, any>>(structuredClone(before));
t.flush(); // drain the opening base batch
t.state.items = ["first"];
t.state.items = ["second"];
t.state.user.name = "dropped";
t.state.user = { id: "u9" };
show("4 writes, 2 survive", t.flush());
}
// ─────────────────────────────────────────────────────────────────────────
// 2. Why checkpointing exists
// ─────────────────────────────────────────────────────────────────────────
console.log("\n2. RECOVERY LENGTH\n");
/**
* A bash-shaped workload: output arrives continuously and the sink writes
* durably on an interval. Each flush is one batch appended to a list.
* On resume the reader folds back to the last BASE batch, so that distance is
* the cost of recovery.
*/
function simulate(writes: number, checkpointEvery: number | null) {
const CAP = 50_000;
const t = track({ out: "" });
const isBaseBatch: boolean[] = [];
for (let i = 0; i < writes; i++) {
const next = `${t.state.out}[${i}] cc -c src/file${i}.c -o build/file${i}.o\n`;
t.state.out = next.length > CAP ? next.slice(next.length - CAP) : next;
if (checkpointEvery !== null && i % checkpointEvery === checkpointEvery - 1) t.rebase();
const ops = t.flush();
if (ops.length > 0) isBaseBatch.push(isBase(ops));
}
const lastBase = isBaseBatch.lastIndexOf(true);
return { written: isBaseBatch.length, replay: isBaseBatch.length - lastBase - 1 };
}
for (const [label, every] of [["never", null], ["rebase() every 50", 50]] as const) {
const r = simulate(500, every);
console.log(` ${label.padEnd(18)} ${r.written} batches written → ${r.replay} to replay on recovery`);
}
console.log(`
Nothing produces a base batch on its own: flush() emits ops, and a replacement
happens only when the producer asks for one. So a stream of appends stays a
stream of deltas indefinitely, and recovery grows without bound.
Two callers need rebase():
- pendingToolOutput, during a long-running command. Without it a ten-minute
build leaves hundreds of batches to fold on resume.
- a facet host on resubscribe. facets.md §9.2 requires the first batch of a
subscription to be a base batch, and the host must be able to produce one
on demand.
`);
// ─────────────────────────────────────────────────────────────────────────
// 3. A replica always agrees
// ─────────────────────────────────────────────────────────────────────────
console.log("3. PRODUCER AND REPLICA AGREE\n");
{
// The whole loop: tracker -> encode -> [boundary] -> decode -> apply.
const t = track<Record<string, unknown>>(structuredClone(before));
const enc = encoder();
const dec = decoder();
let replica: unknown;
const step = (label: string, mutate: () => void) => {
mutate();
const ops = dec.decode(enc.encode(t.flush()));
if (ops.length === 0) return;
// No base-batch branch: apply handles `r` by replacing, and tolerates an
// undefined target because a root op never reads it.
replica = apply(replica, ops);
const same = JSON.stringify(replica) === JSON.stringify(t.target);
console.log(` ${label.padEnd(22)} ${ops.length} op(s) ${isBase(ops) ? "BASE " : "delta"} matches: ${same}`);
};
step("opening base batch", () => {});
step("append to a string", () => { (t.state.note as string); t.state.note = `${t.state.note}!`; });
step("push", () => { (t.state.items as string[]).push("c"); });
step("nested set", () => { (t.state.user as Record<string, unknown>).name = "cy"; });
step("delete", () => { delete t.state.note; });
step("whole replacement", () => { t.state = structuredClone(after); });
step("rebase", () => { t.rebase(); });
}
console.log();

617
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.md vendored Normal file
View File

@@ -0,0 +1,617 @@
# Delta Tracking and the Op Vocabulary
> **Production status:** landed in `packages/chord/src/delta/index.ts`, with tests in `packages/chord/test/delta.test.ts`. Chord owns the dependency-free `Op`/`WireOp`, tracker, applier, codec, and validation boundary; Session storage, the Harness, and facets consume it. The implementation and tests beside this document are historical prototype and benchmark evidence, not production source.
>
> The landed tracker computes deltas at `flush()` from a dirty tree and baseline rather than retaining one op per mutation. That resolves FINDINGS D1. Production remeasurement also closes FINDINGS D2: the generic string path is below surrounding replication/rendering costs, so an explicit append/truncate API is rejected ([decision](append-decision.md)).
One mechanism covers assistant partials, tool output, tool details, lane state, and arbitrary facet state — on the wire and in durable storage.
```bash
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/delta.test.ts
# run from packages/chord
```
## 1. Why not an existing library
Immer, Valtio, Mutative and Colyseus all **record effects, not intent**. A string
is a leaf, so `text += chunk` is one write of the whole new string; there is no
representation in which the library could know you appended. `unshift` is the
same story — the engine shifts every index, so each is a write.
Measured, all four:
| operation | what they emit |
| --- | --- |
| `text += "x"` on 40 KB | `replace` carrying 40 KB |
| `arr.unshift(x)` on 100 items | ~100 index writes |
| `arr.pop()` | a write to a `length` path, which is not a document location |
Immer's own pitfalls page states the guarantee: patches are correct and
**explicitly not minimal**. Colyseus documents the same array weakness — removing
the first of 20 items costs 38 extra bytes.
Diffing base against result ourselves does not fix it either. A differ sees two
values, not intent. Prefix comparison catches pure growth, but a rolling window
that drops from the front *and* appends is neither a prefix nor a suffix of the
old value; measured on 5000 elements the fallback was
`{index: 0, remove: 5000, items: 5000}` — a full replace, in exactly the case the
optimisation existed for.
So the win is not a blind whole-value differ. The tracker records which paths and array operations became dirty, then compares only those subtrees against its accepted baseline at `flush()`.
## 2. Ops
**Tuples are the form. Everywhere. There is no keyed variant.**
```ts
export type Seg = string | number;
export type Path = readonly Seg[];
/** Non-empty: `s`/`d`/`a`/`t` cannot target the root. Enforced by the type. */
export type NonEmptyPath = readonly [Seg, ...Seg[]];
export type Op =
| readonly ["r", JsonValue] // replace the value
| readonly ["s", NonEmptyPath, JsonValue] // set
| readonly ["d", NonEmptyPath] // delete
| readonly ["a", NonEmptyPath, string] // append
| readonly ["t", NonEmptyPath, number] // truncate, in chars
| readonly ["p", Path, number, number, JsonValue[]] // splice: index, remove, items
```
Interning, id references and omitted paths are **not** here — they live in
`WireOp` and exist only between `encode` and `decode` (§4).
**`r` is the only op that replaces a whole value.** It is an op rather than a
frame kind for the same reason `a` and `t` exist: they are specialisations of `s`
that are smaller and state intent.
Do not encode a replacement as a set at the root path. Base-batch detection is a
**correctness boundary** — recovery stops there (§9) — so it must be a token
comparison, `ops[0]?.[0] === "r"`, and not a path inspection. A path-inspecting
predicate has to handle the two-element arity form, where `op[1]` is a value
rather than a path, and misclassifies `["s", []]`.
**Only `p` may target the root**, and only because a tracked value can itself be
an array — `entries.push(x)` on a root array is `["p", [], 3, 0, [x]]`. A `p`
covering its entire target is normalised at flush time to `r` (root) or `s`
(nested), so a root `p` is always a partial modification. The other four verbs
take a `NonEmptyPath`: `["s", [], v]` and `["d", []]` do not typecheck.
**Do not add a keyed in-memory shape with a codec at the serialization boundary.**
Two representations of one thing means two appliers, two size estimates, and a
conversion nobody needs. Ops are tuples in memory, on the wire, and on disk;
readability is what a debug formatter is for.
Six verbs. No `move`, `copy` or `test`: the first two are size optimisations for
a case we do not have, and `test` belongs to a conflict model we do not have,
since there is exactly one authoritative writer.
`chars` counts UTF-16 code units, matching `String.prototype.slice`. **Not bytes.**
Byte caps are a producer concern and never cross a boundary.
The format is JSON-Patch-*shaped*, not RFC 6902 conformant. That costs nothing:
RFC 6902 has had no successor since 2013, still has the same six ops, and nothing
anywhere adds a string splice. Immer was never conformant either.
## 3. The tracker
State is **plain TypeScript**. No containers, no handles, no schema, no
decorators. You mutate normally.
```ts
const t = track(laneView);
t.state.operation.streamingMessage.content[0].text += delta; // -> append
t.state.transcript.push(entry); // -> splice
t.state.tools[0].details.failures.push({ name, msg }); // -> splice
delete t.state.config.model; // -> delete
const ops = t.flush();
```
Three mechanisms, one per case:
**Objects** — ordinary `set` / `deleteProperty` traps.
**Arrays — mutator methods intercepted in the `get` trap.** `arr.push(x)` passes
through `get(arr, "push")` first, so the tracker returns its own function that
records `splice(len, 0, [x])` and then delegates. Intent is captured *before* the
engine performs its index writes, which is why `unshift` is one op rather than
O(n). This is the thing no other library does.
**Strings — dirty marking on the `set` trap, overlap detection at `flush()`.** Strings are primitives, so there is nothing to intercept. Given the accepted `prev` and final `next`, find the longest suffix of `prev` that is a prefix of `next`:
```
overlap === prev.length -> append(next.slice(overlap))
overlap > 0 -> truncate(prev.length - overlap) + append(rest)
overlap === 0 -> set(next)
```
Always correct, because the overlap is **verified**, not guessed. The rolling
window works:
```ts
s.out = s.out.slice(4) + "!!!";
// ["truncate", ["out"], 4]
// ["append", ["out"], "!!!"]
```
### 3.1 Overlap must not use a hand-written KMP
A KMP failure function is asymptotically right and **47x slower in practice**,
because it runs character by character in JS. Measured on 2000 window slides over
a 50 KB string: 2870 ms for KMP, 61 ms for a native probe.
The probe: `indexOf` a short head of `next` within `prev`, verify each candidate
with `endsWith`. Both are native. Verified identical to KMP across 200 window
slides.
A `startsWith` fast path runs first, so a pure append — the dominant case — skips
the probe entirely.
### 3.2 Rules the implementation must hold
**Inserted values are adopted; emitted payloads are cloned at flush.** Callers may retain read-only references but must not mutate adopted objects outside the tracker. `flush()` clones payloads and advances a cloned accepted baseline, so emitted ops never alias producer state.
**`x = undefined` normalises to `delete`.** JSON has no `undefined`, and a `set`
with a missing value is indistinguishable from a lost value after a
`JSON.stringify` round trip.
**`arr.length = n` is translated, never emitted as a path.** Shrink becomes a
splice, grow becomes a splice of nulls. `length` is a real mutation but not a
document location — this is the bug Immer ships (issue 208). Dropping it silently,
as an early prototype did, means `arr.length = 0` never reaches the replica.
**Numeric array keys are normalised to numbers.** Proxy traps deliver `"0"`;
paths carry `0`. Valtio ships the string form and it is worse for both size and
comparison.
**Proxies are cached per object and path**, so `s.a === s.a` and proxies are not
rebuilt on every access.
**Scope is `JsonValue` only, and it must be CHECKED, not assumed.**
`structuredClone` is not a JSON check — it clones `Map`, `Set`, `Date` and
`RegExp` happily, and the result then differs from what `JSON.stringify` produces,
so producer and replica diverge silently. Assert the value at record time and
reject symbol keys. See §7.4 for what leaks without it.
**Exactly two ops reach the root**, and the applier's root branch must handle
both: `r` always, and `p` when the tracked value is itself an array.
`s`/`d`/`a`/`t` cannot, by type, which makes the branch's fallthrough provably
unreachable. Handling only `r` there passes every hand-written test and fails on
the first randomised sequence that produces a root splice.
**Two writes to one address inside a transaction chain.** When several writes to
the same address are emitted in one commit, each must be recorded against the
previous one's result, not against pre-transaction state. The applier replays them
in order; anything else applies the second against a stale base. This did not fire
in the workload that surfaced it — it would have corrupted silently when it did.
### 3.2.1 Replacing the whole value
Assigning to `state` replaces it:
```ts
tracker.state = next; // emits ["r", next]; discards ops recorded before it
```
`state` **must be a setter on the tracker**, not a plain property. Without one,
`tracker.state = next` swaps the proxy for a plain object and every later
mutation is silently untracked — no error, no ops, `target` unchanged. It is also
the first thing anyone tries.
Prior ops are discarded because they describe a value that no longer exists.
Only a *full* replacement collapses to `r`. Rewrite part of the value and the ops
survive, because they are genuinely cheaper:
```
partial rewrite 2 op(s), base=false
["s",["user"],{"id":"u2","name":"bob"}]
["s",["items"],["x"]]
```
### 3.2.2 The first flush is a base batch
`track()` opens a stream, and a consumer starts with nothing, so the first flush
always emits `["r", value]` — carrying any mutations made before it:
```ts
const t = track({ x: 0, l: [] });
t.state.x = 100;
t.state.l.push("xyz");
t.flush(); // [["r", { x: 100, l: ["xyz"] }]]
t.state.x = 101;
t.flush(); // [["s", ["x"], 101]]
```
Requiring the producer to remember a `rebase()` first would fail at runtime, in
the consumer, far from the mistake.
### 3.2.3 Forcing a base batch later
```ts
tracker.rebase(); // next flush is ["r", value]; value unchanged
```
Discarding the pending ops is correct: the proxy mutates the target directly, so
the value already carries them. `tracker.state = tracker.state` has the same effect in the landed implementation, but `rebase()` states the intent directly.
**Nothing produces a base batch on its own.** `flush()` emits ops; a replacement
happens only when the producer asks for one. So a stream of appends stays a
stream of deltas indefinitely.
Recovery replays from the last base batch (§9). From `delta.examples.ts`, a
bash-shaped workload of 500 durable writes into a 50 KB rolling window:
| | batches written | to replay on recovery |
| --- | --- | --- |
| never | 500 | **499** |
| `rebase()` every 50 | 500 | 0 |
Two callers need this:
- **A durable sink.** `pendingToolOutput` accumulates one batch per checkpoint for
as long as a command runs. A `make -j8` running for ten minutes otherwise leaves
hundreds of batches to fold on resume. Checkpointing every N makes "at most N
batches to replay" a policy rather than an accident.
- **A facet host on resubscribe.** `facets.md` §9.2 requires the first batch of
every subscription to be a base batch, and a consumer that sees a gap
resubscribes to get one. The host has to be able to produce one on demand rather
than wait for one to happen by chance.
### 3.3 Known gaps
- `sort` / `reverse` / `fill` / `copyWithin` mark the array dirty and emit the resulting structural/index changes rather than preserving the producer's method intent. Add a dedicated op only if a measured workload needs one.
- A manual index-shift loop (`for (…) a[i] = a[i+1]`) can still cost O(n) sets. Correct, not minimal, and unavoidable — the producer genuinely wrote every element.
- Rolling-window string assignment still runs overlap discovery at flush, deliberately: production measurements do not justify text-specific tracker state or API surface ([decision](append-decision.md)).
### 3.4 Constraints the string algorithm must hold
Each of these is cheap to get wrong in a way that passes hand-written tests.
**A fixed-length probe cannot find an overlap shorter than the probe.** The head
must actually occur in `a`: `"abcdefgh"` -> `"defghxyz"` overlaps by 5, and a
64-character head cannot occur in an 8-character string. Try a long head first,
then fall back to a one-character head. A test using only 50 KB strings, where
overlaps are huge, will pass against a broken implementation.
**Bound the candidate scan.** Repetitive output — a build log, or a run of one
character — makes a long head match at thousands of positions, each costing a full
`endsWith`. Unbounded, 2000 slides over a 50 KB window take 4.7 s against 93 ms
bounded. Exceeding the bound returns 0, which emits a set: larger, never wrong.
**Property tests before anything depends on this.** 3000 random mutation
round-trips find the root-splice hole in §3.2 on the first run; the hand-written
cases do not. `delta.test.ts` is the suite to port.
## 4. The codec
Two vocabularies, not one.
`Op` is what the tracker produces and `apply` consumes. **Paths are always
inline.** It knows nothing about a dictionary.
`WireOp` is what crosses a boundary. It adds exactly two compressions:
```
["#", id, path] defines an id, emitted on a path's SECOND use
a numeric PathRef references a previously defined id
a shortened tuple reuses the previous op's path; arity disambiguates
```
`encoder().encode(ops): WireOp[]` and `decoder().decode(wire): Op[]` are the only places either exists. Keeping them out of `Op` means `apply` has no id resolution, no `#`
case, and no previous-path state — three branches removed from the hot path —
and dirty-tree generation always works with inline paths before encoding.
`["r", value]` carries no path, so it encodes to itself. That is why `isBase`
works unchanged on either vocabulary.
### 4.1 Two rules that are easy to get wrong
**One encoder/decoder pair per stream.** The id table spans a whole subscription
or file: a path interned in batch 3 is referenced in batch 40. A second consumer
that subscribes at batch 40 has never seen the definition, so it needs its own
encoder. Sharing one across consumers hands the late subscriber ids it cannot
resolve, and a base batch does not rescue it — `["r", value]` carries no refs and
leaves the table empty.
**Reset the table on a base batch.** A reader replays from the *last* base batch
with a fresh decoder, so everything after one must be self-contained. Carrying
ids across a replacement emits references to definitions the reader never saw:
```
4 : [["a",0,"x4"],["a",1,"y4"]]
5 BASE: [["r",{…}]]
6 : [["a",0,"x6"]] <- id 0 was defined in batch 1
-> RECOVERY FAILED: PathError - unresolvable path: 0
```
**Arity omission is scoped to a batch.** Letting it span batches makes a batch's
first op depend on the previous batch's last one, so a reader that skips or
reorders a batch decodes into the wrong path. Ids are the only cross-batch state,
and the dictionary makes those explicit.
### 4.2 Intern on second use, not first
A definition costs more than the path it replaces, so interning on first use
loses on every path written exactly once — and most are. Measured on a bulk
workload: **255.5 KB first-use versus 179.6 KB with no interning at all.**
### 4.3 Measured
Wire bytes against inline paths, round-trip verified:
| stream | inline | wire | saved |
| --- | --- | --- | --- |
| one hot path (a rolling tool-output window) | 11,290 B | 7,538 B | **33.2%** |
| four alternating paths (lane state) | 14,160 B | 11,032 B | **22.1%** |
| 200 distinct paths | 16,455 B | 16,035 B | 2.6% |
Omission does the work in the first case, interning in the second. In the third
neither helps and the definitions cost a little — which is the case second-use
interning exists to bound.
## 5. Flush emits ops, and drops the dead ones
`flush()` computes ops for the dirty paths against the last accepted baseline. It does not retain one op per mutation, so repeated and interleaved writes stay bounded by changed state rather than write count. There is no size comparison or replacement heuristic: a replacement is something the producer asks for by assigning `state` or calling `rebase()`.
An earlier design compared op bytes against the value's serialised size and
replaced when ops were larger. It was removed. Measured against emitting ops
unconditionally, on six workloads, it changed the output in two — both requiring
hundreds of distinct paths to change in a single flush. Neither the harness nor
the replication path does that: `LaneSnapshot` is folded one event at a time, and
the widest case (`run_end`) touches four fields. The rule cost a per-flush
comparison, a running size estimate, and an invalidation rule for the ops that
could not maintain it.
### 5.1 Dirty-tree collapse
The landed tracker records only which subtrees are dirty. At flush it compares each dirty subtree's accepted baseline with its final value and emits the surviving structural change. Repeated writes to one field collapse naturally; alternating writes to two fields retain two dirty paths rather than one op per mutation; a parent replacement subsumes dirty descendants where the final comparison permits it. `packages/chord/test/delta.test.ts` pins the interleaved rolling-window case at at most three ops after 1,000 alternating writes.
This replaces the prototype's backwards dead-op pass and adjacent-only coalescer. Do not port those algorithms into Chord: flush-time generation is the D1 fix.
> **Object key order is not a replicated invariant.** Values round-trip, but delete-and-reinsert activity within one flush can produce a different insertion order on a replica. Do not hash or content-address a replicated value, and sort explicitly where display order matters.
## 6. There is no frame type
The logical batch is `Op[]`; transport and durable storage carry the statefully encoded `WireOp[]`. Do not wrap either batch.
**`seq` does not belong in the payload.** It would defend against a lossy
transport we do not have: a durable list element already carries `seq` from
storage, an in-process callback cannot skip, and SSE either delivers in order or
breaks — and a break means resubscribe, which means a base batch. A second copy of
storage's `seq` could only disagree with the first. The SSE binding stamps `id:`,
which is where transport metadata belongs.
**Nothing else needs wrapping either.** A `kind` discriminator is redundant once a
replacement is an op (§2), and the address already says which value a batch
belongs to. What would remain is a struct around an array.
A producer batch is just `Op[]`, and its encoded boundary form is `WireOp[]`. A **base batch** is one whose first op is `r`;
`isBase(ops)` is `ops[0]?.[0] === "r"`, exact rather than heuristic because flush
guarantees `r` appears at index 0 or not at all (§5).
Everything a frame used to do is now done by something that already existed:
| was on the frame | now |
| --- | --- |
| `seq` | the list element's `seq`, or the SSE `id:` |
| `kind: "replace"` | the `r` op |
| which value it belongs to | the address |
| "this is a snapshot" | the `"base"` storage tag |
**Resubscription is a base batch plus buffered batches** — the same thing the lane
adapter already does for the harness: snapshot first, then whatever accumulated
while the client caught up. There is no `Last-Event-ID` and no cross-subscription
resume; that would need a retained op log, which §5 deliberately does not keep.
A consumer that sees a gap, fails to resolve a path, or connects cold takes one
path: ask for a base batch. That is why there is no `Rebase` type.
The cost, stated plainly: **a consumer cannot detect staleness from the payload
alone.** It relies on the transport reporting a break. For SSE and in-process that
is sound. If a lossy or multiplexed transport ever appears, sequencing goes on that
transport's envelope — not back into the ops.
## 7. Safety
Ops arrive from a facet, a plugin compartment, or a tool whose details may echo
model output. **None of it is trusted input**, and the boundary where untrusted
data meets trusted machinery is where this design has to hold.
The reference point is CVE-2025-55182 — RCE in React Server Components, CVSS 10.0,
exploited in the wild. Their Flight protocol is a compact tagged wire format that
reconstructs structure, so the resemblance is real. The failure was *"fails to
validate the structure correctly... treats the fake object as genuine"*: a forged
Chunk resolved as a Promise and exposed internal state containing gadgets to reach
`Function`.
**We are structurally safer, and not because of diligence:**
| | Flight | ops |
| --- | --- | --- |
| can describe runtime objects | yes — Chunks resolve as Promises | no |
| can reference code or modules | yes — client components | no |
| values | arbitrary object graphs | `JsonValue` |
| worst case from a forged payload | RCE | corrupted replica state |
Flight *must* reference code; that is its job. An op can only put a `JsonValue` at
a path, so there is no first link in a gadget chain. This is why an attacker cannot
plant a non-resolving `then` on `Object.prototype`: a data-only `then` is not
callable, and `await` resolves normally when `then` is not callable.
> **Rule: no op may name something the host resolves.** Not a mutation name, not a
> component id, not a module reference. Mutation names were removed for an
> unrelated reason (§8); this is the second and better reason never to reintroduce
> them, because it is the ingredient that made Flight exploitable.
### 7.1 Paths are the dangerous part
`JSON.parse` is safe on its own — `{"__proto__":{}}` becomes an *own* property.
The hazard is `parent[key]`, which is exactly what applying a path does.
A single key is not enough. `x["__proto__"] = v` swaps *x's own* parent, which is
local. It takes a **walk** — `x["__proto__"]["polluted"] = v` — to reach the shared
prototype, and a path is precisely a walk.
`constructor` is worse than `__proto__`, because `({}).constructor.constructor` is
`Function`. That ladder is closed here only because op values cannot be functions;
it is closed properly by rejecting the segment.
**Reserved path segments: `__proto__`, `constructor`, `prototype`.** Rejected at
record time *and* at apply time, including through an interned path id.
They are reserved as *segments*, not as values. An object with a literal
`"__proto__"` key replicates fine as a whole value; only a path walking *through*
it is refused. This is a genuine restriction on what is mutable — document it, do
not pretend it away.
Two further measures on the same hazard:
- **Write with `Object.defineProperty`**, never assignment, so an inherited setter
cannot run.
- **Resolve own properties only** (`Object.hasOwn`), so a walk cannot escape into
the prototype chain and an inherited getter cannot fire. This blocks
`constructor` a second time, since it is inherited.
What prototype pollution actually buys an attacker here is narrower than it first
appears: **it flips defaults, it does not overwrite explicit values.** An own
property shadows. So `{name:"bob", isAdmin:false}` is unaffected; an option bag read
as `opts.skipSandbox` is not. Our own `ShellExecOptions`, `ShellOutputCaptureOptions`,
`ListReadOptions` and `TrackerOptions` are exactly that shape. Note also that
`"x" in {}` and `const {x = false} = {}` both lie under pollution — which is why
the applier uses `Object.hasOwn`.
### 7.2 Array indices
An index may address an existing element or append exactly one past the end.
This is not an arbitrary cap; it is what keeps the value a `JsonValue`. A sparse
array does not survive a JSON round trip — holes serialise to `null` and return as
real properties — so `arr[7] = x` on a length-3 array already produces state a
replica cannot match. Rejecting the write is more honest than diverging.
It removes a denial of service as a side effect rather than as its purpose:
`["s",["xs",4294967290],1]` would otherwise allocate a 4.29-billion-entry array
from one op. Growth stays available and stays proportional, because
`arr.length = n` is emitted as a splice of explicit nulls whose op size grows with
the gap.
### 7.3 Validate op structure on decode
This is the RSC lesson applied to us. A decoder must not trust tuple shape.
Measured against an unvalidated applier:
| malformed op | result |
| --- | --- |
| `["p",["xs"],0,0,"not-an-array"]` | string spread into the array: `["n","o","t",…]` |
| `["s","a",9]` — path is a string | accepted; `"a".slice(0,-1)` is `""`, so it wrote at the root |
| `["ZZZ",["a"],9]` | silently ignored, replica diverges with no error |
Validate: verb is known, arity matches the verb, a path is an array of strings and
non-negative integers, `p` carries integer index and count plus an array of items,
`#` defines an array path. An unknown verb is an **error**, not a no-op — silently
skipping it is how a newer producer's op disappears and a replica drifts.
**One validator per vocabulary.** `assertValidOp` guards `apply` and rejects ids,
short forms and `#`; `assertValidWireOp` guards `decode` and permits them.
Validating a decoded op against the wire grammar is laxer than its own type: a
two-element `["s", value]` would pass, and `apply` would then read the value as a
path.
### 7.4 The tracker's cage leaks on types, not shape
A facet cannot forge an op: it mutates plain objects and the tracker builds the
tuples, so shape is well-formed by construction. Values and keys are another
matter, and `structuredClone` is **not** a JSON check.
| what a facet writes | what happens |
| --- | --- |
| a function | throws (`DataCloneError`) |
| a BigInt, a cycle | throws |
| `new Map([[1,2]])` | op carries `{}`, producer keeps a real Map — **silent divergence** |
| `new Date(0)` | op carries an ISO string, producer keeps a Date |
| `state[Symbol("s")] = 1` | emits `["s",[null],1]` — **a malformed op from our own tracker** |
So the tracker must check what it is given, at record time, where the mistake is:
- **Reject symbol keys.** They are not path segments.
- **Assert `JsonValue` on every recorded value.** Reject `Map`, `Set`, `Date`,
`RegExp`, typed arrays and class instances explicitly. The scope restriction in
§3.2 was a comment; it has to be a check.
A getter on state is safe — the trap records the computed result — and a facet
value that *looks* like an op is nested inside `["s", path, value]` and can never
be read as a top-level op, because nothing flattens.
### 7.5 Applier
```ts
export function apply<T>(target: T | undefined, ops: readonly Op[]): T;
```
Six verbs, no domain knowledge, no library, no tool code, no registry lookup, and
**no path table** — ids and omitted paths are resolved by `decode` before `apply`
sees anything (§4).
It returns the value rather than mutating in place, because `r` replaces it
outright.
Runs against a **plain mutable object the consumer owns**. It must not be pointed
at a value produced by Immer, which deep-freezes and would throw.
A page of code in any language, which is the property that keeps non-JS consumers
viable. Colyseus ships decoders for C#, Lua and Haxe on the same basis.
## 8. What this removes from the codebase
Concrete deletions, not simplifications in principle:
- **Immer**, from the harness and from the facet layer. See §1 for why a
patch-producing library cannot serve this purpose.
- **Per-type reducers.** There is one applier, `apply(target, ops)`, with no
domain knowledge and no registry. Nobody writes a fold for `ToolOutputState` or
for a plugin's shape.
- **`detailMutations` and `initialDetails`**, and with them the argument that
`details: unknown` forces special handling. A structural tracker never needs
the type.
- **`Rebase` as a reducer return value.** A fold that cannot apply leaves state
alone and the host sends a base batch.
- **Mutation names on the wire**, and therefore mutation-name version skew. Names
never cross a boundary, so adding or renaming one is not a breaking change.
And three things not to build, each of which looks reasonable until §1:
- **A blind whole-value differ.** The landed tracker compares only dirty subtrees against its accepted baseline; it does not scan unrelated state.
- **A keyed op shape plus a codec.** Tuples are the form everywhere (§2).
- **A frame wrapper.** What travels is `Op[]` (§6).
## 9. Durable form
A tracked value is stored as a **list of encoded `WireOp[]` batches**, one appended per flush. Base batches carry the storage tag `"base"`; one stateful decoder per value decodes them before `apply`.
Recovery reads backwards to the last base batch and applies forward:
```ts
readList(address, { order: "desc", stopAtTag: "base", limit: 100 })
```
`stopAtTag` is a stop condition *within a page*: if no base batch is in the page,
the consumer pages again with the cursor. The tag lives on the storage record
beside `seq`, never inside the value, so storage never parses ops. See
[scopes.md](../02-scopes/scopes.md) §11.
This is what makes "a replacement truncates recovery" (§5) real rather than
aspirational — a reader stops at the last base batch instead of replaying from
the beginning.
## 10. Open questions
- Prefix interning (a trie over path heads) if a workload emerges with many
distinct paths sharing long prefixes. Second-use interning (§4) removes the
pathological case; this would go further.
- Whether array reordering needs a dedicated op; the landed tracker currently marks the array dirty and emits the resulting structural/index changes.
- Cross-language replicas: the applier is a page of code in any language, but the
interning tables and the wire framing are not specified for a non-JS consumer.
Address interning in the JSONL log is the same trick as path interning, applied
one layer up over `namespace` + `key`. It is **not** an open question — it is
specified in [scopes.md](../02-scopes/scopes.md) §12 and is a separate dictionary from this one.

563
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.test.ts vendored Normal file
View File

@@ -0,0 +1,563 @@
import { describe, expect, it } from "vitest";
import { apply, assertJsonValue, assertValidOp, assertValidWireOp, decoder, encoder, isBase, overlap, track, type Op, type WireOp } from "./delta-impl.ts";
const PAD = "p".repeat(400); // see "adaptive emission": a delta must be able to win
describe("tracker: intent", () => {
it("records an append, not a replacement", () => {
const t = track({ s: "", pad: PAD });
t.flush(); // drain the base batch
t.state.s += "ab";
t.state.s += "cd";
const ops = t.flush();
expect(ops).toEqual([["a", ["s"], "abcd"]]);
expect(apply({ s: "", pad: PAD }, ops)).toEqual({ s: "abcd", pad: PAD });
});
it("recovers truncate+append from a rolling window", () => {
// The case every effect-recording library degrades to a whole-value set.
const t = track({ s: "abcdefgh", pad: PAD });
t.flush(); // drain the base batch
t.state.s = t.state.s.slice(3) + "xyz";
const ops = t.flush();
expect(ops).toEqual([["t", ["s"], 3], ["a", ["s"], "xyz"]]);
expect(apply({ s: "abcdefgh", pad: PAD }, ops)).toEqual({ s: "defghxyz", pad: PAD });
});
it("records array intent as one splice", () => {
const t = track({ xs: [1, 2], pad: PAD });
t.flush(); // drain the base batch
t.state.xs.push(3);
expect(t.flush()).toEqual([["p", ["xs"], 2, 0, [3]]]);
});
it("normalises undefined to a delete", () => {
const t = track({ a: 1, pad: PAD });
t.flush(); // drain the base batch
(t.state as Record<string, unknown>).a = undefined;
expect(t.flush()).toEqual([["d", ["a"]]]);
});
});
describe("tracker: root ops", () => {
it("splices a value that is itself an array", () => {
const t = track([1, 2, 3, PAD]);
t.flush(); // drain the base batch
t.state.push(4);
const ops = t.flush();
expect(ops).toEqual([["p", [], 4, 0, [4]]]);
expect(apply([1, 2, 3, PAD], ops)).toEqual([1, 2, 3, PAD, 4]);
});
it("normalises a splice covering the whole root to a replacement", () => {
const t = track([1, 2, 3]);
t.flush(); // drain the base batch
t.state.splice(0, 3, 9);
const ops = t.flush();
expect(ops).toEqual([["r", [9]]]);
expect(isBase(ops)).toBe(true);
});
it("normalises a nested splice-all to a set, not a replacement", () => {
const t = track({ xs: [1, 2, 3], pad: PAD });
t.flush(); // drain the base batch
t.state.xs.splice(0, 3, 9);
expect(t.flush()).toEqual([["s", ["xs"], [9]]]);
});
it("normalises length = 0 on the root", () => {
const t = track([1, 2, 3]);
t.flush(); // drain the base batch
t.state.length = 0;
const ops = t.flush();
expect(ops).toEqual([["r", []]]);
expect(apply([1, 2, 3], ops)).toEqual([]);
});
});
describe("replacing the whole value", () => {
// The proxy cannot observe `tracker.state = next`: `state` belongs to the
// tracker, not to the tracked object, so assigning to it swaps the proxy for a
// plain object and silently stops tracking. Replacement must be a method.
it("assigning to state emits a base batch and keeps tracking", () => {
const t = track<Record<string, unknown>>({ p: 1, q: PAD });
t.flush(); // drain the base batch
t.state.p = 2;
t.state = { r: 9, s: "new" };
const ops = t.flush();
expect(ops).toEqual([["r", { r: 9, s: "new" }]]);
expect(isBase(ops)).toBe(true);
expect(apply({}, ops)).toEqual(t.target);
t.state.r = 10; // the new value is tracked
expect(t.flush()).toEqual([["s", ["r"], 10]]);
});
it("assigning to state discards ops recorded before it", () => {
// They describe a value that no longer exists.
const t = track<Record<string, unknown>>({ p: 1, q: PAD });
t.flush(); // drain the base batch
t.state.p = 2;
t.state = { z: 1 };
expect(t.flush()).toEqual([["r", { z: 1 }]]);
});
it("a partial rewrite keeps its ops", () => {
const t = track<Record<string, unknown>>({ a: "x".repeat(200), b: "y".repeat(200) });
t.flush(); // drain the base batch
t.state.a = "p";
const ops = t.flush();
expect(isBase(ops)).toBe(false);
expect(ops).toEqual([["s", ["a"], "p"]]);
});
it("rebase() forces a base batch without changing the value", () => {
// This is the checkpoint. Recovery replays from the last base batch, so a
// producer must be able to bound that rather than depend on the adaptive
// size rule, which compares each flush independently.
const t = track<Record<string, unknown>>({ p: 1, pad: PAD });
t.flush(); // drain the base batch
t.rebase();
const ops = t.flush();
expect(isBase(ops)).toBe(true);
expect(apply({}, ops)).toEqual(t.target);
});
it("rebase() applies once, not to every later flush", () => {
const t = track<Record<string, unknown>>({ p: 1, pad: PAD });
t.flush(); // drain the base batch
t.rebase();
t.flush();
t.state.p = 2;
expect(t.flush()).toEqual([["s", ["p"], 2]]);
});
});
describe("the first flush", () => {
it("is always a base batch", () => {
// A consumer starts with nothing, so a stream opening with deltas has
// nothing to apply them to. Requiring the producer to remember would fail
// at runtime, in the consumer, rather than where the mistake is.
const t = track<Record<string, unknown>>({ x: 0 });
t.state.x = 100;
const ops = t.flush();
expect(isBase(ops)).toBe(true);
expect(ops).toEqual([["r", { x: 100 }]]); // carries mutations made before it
});
it("is followed by deltas", () => {
const t = track<Record<string, unknown>>({ x: 0, pad: PAD });
t.flush();
t.state.x = 1;
expect(t.flush()).toEqual([["s", ["x"], 1]]);
});
it("emits nothing when untouched", () => {
// An unmutated tracker has nothing to say; the base batch is owed to a
// consumer, and there is no consumer until someone flushes.
const t = track<Record<string, unknown>>({ x: 0 });
expect(t.flush()).toEqual([["r", { x: 0 }]]);
expect(t.flush()).toEqual([]);
});
});
describe("apply and fan-out", () => {
it("adopts an `r` payload rather than copying it", () => {
// The consumer owns the batch it was handed. Copying every base batch
// defensively doubles the memory of the one op carrying the whole value.
//
// The rule that follows: do not hand one batch to two in-process
// consumers. This test pins the aliasing so nobody "fixes" it by adding a
// clone back.
const t = track<Record<string, unknown>>({ n: 0 });
const batch = t.flush();
const a = apply(undefined, batch) as Record<string, unknown>;
const b = apply(undefined, batch) as Record<string, unknown>;
a.n = 1;
expect(b.n).toBe(1);
});
it("does not alias the producer", () => {
const t = track<Record<string, unknown>>({ x: 0 });
const replica = apply(undefined, t.flush()) as Record<string, unknown>;
t.state.x = 999;
t.flush();
expect(replica.x).toBe(0);
});
it("folds a whole stream without a base-batch branch", () => {
// `apply` handles `r` by replacing, and tolerates an undefined target,
// so a consumer needs no isBase check and no clone of its own.
const t = track<Record<string, any>>({ x: 0, l: [] as string[] });
const enc = encoder();
const dec = decoder();
let replica: unknown;
const send = () => { replica = apply(replica, dec.decode(enc.encode(t.flush()))); };
t.state.x = 100;
t.state.l.push("xyz");
send();
t.state.x = 101;
send();
expect(replica).toEqual(t.target);
});
});
describe("dead-op elimination", () => {
// Ops apply in order, so an op is dead if a later one overwrites the whole
// subtree it lives in. Only s/d/r dominate; a/t/p modify what is there.
const run = (mutate: (s: any) => void, init: any = { a: { b: 1 }, x: 1, y: 2 }) => {
const t = track<any>(structuredClone(init));
t.flush(); // drain the base batch
mutate(t.state);
return t.flush();
};
it("collapses repeated writes to one field", () => {
expect(run((s) => { s.x = 1; s.x = 2; s.x = 3; })).toEqual([["s", ["x"], 3]]);
});
it("drops a write superseded by a later, non-adjacent one", () => {
expect(run((s) => { s.x = 10; s.y = 20; s.x = 30; }))
.toEqual([["s", ["y"], 20], ["s", ["x"], 30]]);
});
it("drops a child write when the parent is replaced after it", () => {
expect(run((s) => { s.a.b = 99; s.a = { c: 5 }; })).toEqual([["s", ["a"], { c: 5 }]]);
});
it("keeps a child write that follows the parent replacement", () => {
expect(run((s) => { s.a = { b: 1 }; s.a.b = 7; }))
.toEqual([["s", ["a"], { b: 1 }], ["s", ["a", "b"], 7]]);
});
it("collapses set-then-delete to the delete", () => {
expect(run((s) => { s.x = 5; delete s.x; })).toEqual([["d", ["x"]]]);
});
it("keeps delete-then-set, to preserve key position", () => {
// Dropping the delete would leave the key in its original slot while the
// producer moved it to the end. Same value, different serialisation.
expect(run((s) => { delete s.x; s.x = 5; })).toEqual([["d", ["x"]], ["s", ["x"], 5]]);
});
it("is linear in the number of ops", () => {
// The naive formulation compares every op against every dominator, which
// is quadratic and degrades on exactly the wide flush this pass cleans up.
const wide = (n: number) => {
const root: Record<string, number> = {};
for (let i = 0; i < n; i++) root[`f${i}`] = i;
const t = track(root);
t.flush(); // drain the base batch
const started = performance.now();
for (let i = 0; i < n; i++) t.state[`f${i}`] = i + 1;
t.flush();
return performance.now() - started;
};
wide(200); // warm
const small = Math.max(wide(250), 0.1);
const large = wide(2500);
expect(large / small).toBeLessThan(40); // linear would be ~10x
});
it("collapses a pathological redundant producer", () => {
const t = track<any>({ a: { b: 1 }, x: 1 });
t.flush(); // drain the base batch
for (let i = 0; i < 2000; i++) { t.state.x = i; t.state.a.b = i; }
t.state.a = { done: true };
expect(t.flush()).toHaveLength(2);
});
});
describe("flush", () => {
it("drops everything before a replacement", () => {
const t = track([1, 2]);
t.flush(); // drain the base batch
t.state.push(3);
t.state.splice(0, 3, 7);
expect(t.flush()).toEqual([["r", [7]]]);
});
it("keeps the prefix when the replacement is nested", () => {
// `s` on a subtree is not a root replacement, so earlier ops stay live.
const t = track({ a: 1, xs: [1, 2, 3], pad: PAD });
t.flush(); // drain the base batch
t.state.a = 2;
t.state.xs.splice(0, 3, 9);
expect(t.flush()).toEqual([["s", ["a"], 2], ["s", ["xs"], [9]]]);
});
it("rejects a constructor walk", () => {
// ({}).constructor.constructor is Function — the classic escape ladder.
expect(() => apply({}, JSON.parse('[["s",["constructor","prototype","gadget"],true]]'))).toThrow();
expect(({} as Record<string, unknown>).gadget).toBeUndefined();
});
it("rejects a forbidden path reached through an interned id", () => {
const dict = new Map<number, readonly (string | number)[]>([[0, ["__proto__", "w"]]]);
expect(() => apply({}, [["s", 0, true]], (id) => dict.get(id)!)).toThrow();
});
it("does not run an inherited setter", () => {
Object.defineProperty(Object.prototype, "trap", {
get() { throw new Error("inherited getter ran"); },
configurable: true,
});
try {
expect(() => apply({}, [["s", ["trap"], 1]])).not.toThrow();
} finally {
delete (Object.prototype as Record<string, unknown>).trap;
}
});
it("allows a reserved name as a VALUE key", () => {
// Reserved as segments, not as values.
const out = apply({}, [["s", ["a"], JSON.parse('{"__proto__":{"z":1}}')]]);
expect(({} as Record<string, unknown>).z).toBeUndefined();
expect(out).toBeDefined();
});
});
describe("safety: array indices", () => {
// An index may address an existing element or append exactly one past the end.
// Not an arbitrary cap: a sparse array does not survive a JSON round trip, so
// arr[7] = x on a length-3 array already produces unreplicable state.
it("writes an existing index", () => {
expect(apply({ xs: [1, 2, 3] }, [["s", ["xs", 1], 9]])).toEqual({ xs: [1, 9, 3] });
});
it("appends one past the end", () => {
expect(apply({ xs: [1, 2, 3] }, [["s", ["xs", 3], 9]])).toEqual({ xs: [1, 2, 3, 9] });
});
it("rejects a gap", () => {
expect(() => apply({ xs: [1, 2, 3] }, [["s", ["xs", 5], 9]])).toThrow();
});
it("rejects a huge index", () => {
// Would otherwise allocate 4.29 billion entries from one op.
expect(() => apply({ xs: [] }, [["s", ["xs", 4_294_967_290], 1]])).toThrow();
});
it("allows explicit growth with nulls", () => {
expect(apply({ xs: [1] }, [["p", ["xs"], 1, 0, [null, null, 9]]])).toEqual({ xs: [1, null, null, 9] });
});
});
describe("safety: op structure", () => {
// The CVE-2025-55182 lesson: a decoder must not trust tuple shape.
it("rejects an unknown verb rather than skipping it", () => {
// Silently skipping is how a newer producer's op vanishes and a replica drifts.
expect(() => apply({ a: 1 }, [["ZZZ", ["a"], 9] as never])).toThrow();
});
it("rejects non-array splice items", () => {
// Unvalidated, this spreads a string: ["n","o","t","-","a",…]
expect(() => apply({ xs: [1] }, [["p", ["xs"], 0, 0, "not-an-array"] as never])).toThrow();
});
it("rejects a string path", () => {
// Unvalidated, "a".slice(0,-1) is "", so it resolved to the ROOT and wrote.
expect(() => apply({ a: 1 }, [["s", "a", 9] as never])).toThrow();
});
it("rejects a non-tuple op", () => {
expect(() => apply({ a: 1 }, [{ op: "s" } as never])).toThrow();
expect(() => apply({ a: 1 }, [null as never])).toThrow();
});
it("clamps a splice remove past the end, as Array.prototype.splice does", () => {
// Deterministic and identical on both sides — not a hole.
expect(apply({ xs: [1, 2] }, [["p", ["xs"], 0, 1e9, []]])).toEqual({ xs: [] });
});
});
describe("safety: what a facet can put on the wire", () => {
// A facet cannot forge an op — it mutates plain objects and the tracker builds
// the tuples. Values and keys are the leak, and structuredClone is NOT a JSON
// check: it clones a Map happily, and the op then carries {} while the producer
// keeps a real Map.
const rejects = (name: string, mutate: (s: Record<string, unknown>) => void) =>
it(`rejects ${name}`, () => {
const t = track<Record<string, unknown>>({ pad: PAD });
t.flush(); // drain the base batch
expect(() => mutate(t.state)).toThrow();
});
rejects("a function", (s) => { s.f = () => {}; });
rejects("a Map", (s) => { s.m = new Map([[1, 2]]); });
rejects("a Date", (s) => { s.d = new Date(0); });
rejects("a BigInt", (s) => { s.b = 1n; });
rejects("a cycle", (s) => { const a: Record<string, unknown> = {}; a.self = a; s.c = a; });
rejects("a symbol key", (s) => { s[Symbol("k") as unknown as string] = 1; });
it("allows a getter on state, recording the computed result", () => {
const t = track<Record<string, unknown>>({ pad: PAD });
t.flush(); // drain the base batch
Object.defineProperty(t.target, "g", { get: () => 42, enumerable: true });
t.state.x = 1;
expect(t.flush()).toEqual([["s", ["x"], 1]]);
});
it("accepts a value that merely looks like an op", () => {
// Nested inside ["s", path, value]; nothing flattens, so it cannot be read
// as a top-level op.
const t = track<Record<string, unknown>>({ pad: PAD });
t.flush(); // drain the base batch
t.state.x = ["r", { evil: true }];
expect(t.flush()).toEqual([["s", ["x"], ["r", { evil: true }]]]);
});
});
describe("assertions", () => {
it("assertJsonValue rejects non-JSON", () => {
for (const bad of [new Map(), new Set(), new Date(), /re/, () => {}, Number.NaN, Infinity]) {
expect(() => assertJsonValue(bad)).toThrow();
}
for (const good of [null, true, 1, "s", [1, "a"], { a: { b: [null] } }]) {
expect(() => assertJsonValue(good)).not.toThrow();
}
});
it("assertValidOp accepts decoded ops and rejects wire forms", () => {
// Validating Op against the wire grammar would be laxer than the type: a
// two-element ["s", value] would pass, and apply would read the value as a
// path. Each vocabulary gets the validator that matches it.
for (const op of [
["r", { a: 1 }], ["s", ["a"], 1], ["d", ["a"]],
["a", ["a"], "x"], ["t", ["a"], 2], ["p", ["a"], 0, 0, []],
] as never[]) {
expect(() => assertValidOp(op)).not.toThrow();
}
for (const wireOnly of [["s", 1], ["d"], ["a", "x"], ["t", 2], ["p", 0, 0, []], ["#", 0, ["a"]], ["s", 0, 1]] as never[]) {
expect(() => assertValidOp(wireOnly)).toThrow();
expect(() => assertValidWireOp(wireOnly)).not.toThrow();
}
});
});
describe("codec: path interning and arity omission", () => {
// One pair per stream. The table spans a whole subscription or file, so a
// second consumer joining later needs its own encoder.
const roundTrip = (batches: Op[][]) => {
const enc = encoder();
const dec = decoder();
return batches.map((ops) => dec.decode(enc.encode(ops)));
};
it("round-trips a stream exactly", () => {
const t = track<Record<string, any>>({ a: { deep: "" }, b: { deep: "" } });
t.flush(); // drain the base batch
const batches: Op[][] = [];
for (let i = 0; i < 6; i++) {
t.state.a.deep += `x${i}`;
t.state.b.deep += `y${i}`;
batches.push(t.flush());
}
expect(roundTrip(batches)).toEqual(batches);
});
it("interns on second use, not first", () => {
const enc = encoder();
const p = ["a", "deep"] as const;
const first = enc.encode([["a", p, "1"]]);
const second = enc.encode([["a", p, "2"]]);
expect(first).toEqual([["a", p, "1"]]); // inline
expect(second).toEqual([["#", 0, p], ["a", 0, "2"]]); // define, then use
});
it("omits the path when it repeats", () => {
const enc = encoder();
const p = ["a"] as const;
expect(enc.encode([["s", p, 1], ["s", p, 2]])).toEqual([["s", p, 1], ["s", 2]]);
});
it("resets the table on a base batch, so recovery is self-contained", () => {
// A reader replays from the LAST base batch with a fresh decoder. Ids
// defined before it were never seen, so keeping them breaks recovery with
// an unresolvable path id.
const enc = encoder();
const p = ["a", "deep"] as const;
enc.encode([["a", p, "1"]]);
enc.encode([["a", p, "2"]]); // id 0 assigned
const base = enc.encode([["r", { a: { deep: "x" } }]]);
const after = enc.encode([["a", p, "3"]]);
expect(base).toEqual([["r", { a: { deep: "x" } }]]);
expect(after).toEqual([["a", p, "3"]]); // inline again
const dec = decoder();
expect(() => dec.decode(base)).not.toThrow();
expect(dec.decode(after)).toEqual([["a", p, "3"]]);
});
it("survives recovery from the last base batch", () => {
const enc = encoder();
const t = track<Record<string, any>>({ a: { deep: "" }, b: { deep: "" } });
t.flush(); // drain the base batch
const wire: WireOp[][] = [];
for (let i = 0; i < 8; i++) {
t.state.a.deep += `x${i}`;
t.state.b.deep += `y${i}`;
if (i === 5) t.rebase();
wire.push(enc.encode(t.flush()));
}
const lastBase = wire.map(isBase).lastIndexOf(true);
const dec = decoder();
let replica: unknown;
for (const w of wire.slice(lastBase)) {
const ops = dec.decode(w);
replica = replica === undefined
? structuredClone((ops[0] as ["r", unknown])[1])
: apply(replica, ops);
}
expect(replica).toEqual(t.target);
});
it("round-trips random streams", () => {
for (let round = 0; round < 300; round++) {
const t = track<Record<string, any>>({ a: { p: "", q: "" }, b: [] as unknown[], c: 0 });
t.flush(); // drain the base batch
const batches: Op[][] = [];
for (let i = 0; i < 8; i++) {
const r = Math.random();
if (r < 0.3) t.state.a.p += "x";
else if (r < 0.5) t.state.a.q += "y";
else if (r < 0.65) t.state.b.push(i);
else if (r < 0.8) t.state.c = i;
else if (r < 0.9) delete t.state.c;
else t.rebase();
const ops = t.flush();
if (ops.length > 0) batches.push(ops);
}
expect(roundTrip(batches)).toEqual(batches);
}
});
});
describe("property: random round-trip", () => {
// 3000 random sequences found the root-splice hole on the first run, before any
// hand-written case did.
const rnd = (d = 0): unknown => {
const r = Math.random();
if (d > 2 || r < 0.3) return Math.floor(Math.random() * 5);
if (r < 0.45) return ["x", "y", null, true][Math.floor(Math.random() * 4)];
if (r < 0.7) return Array.from({ length: Math.floor(Math.random() * 4) }, () => rnd(d + 1));
const o: Record<string, unknown> = {};
for (const k of ["a", "b", "c"]) if (Math.random() < 0.6) o[k] = rnd(d + 1);
return o;
};
it("a replica always matches the producer", () => {
let checked = 0;
for (let i = 0; i < 3000; i++) {
const base = rnd();
if (typeof base !== "object" || base === null) continue;
const t = track(structuredClone(base) as object);
t.flush(); // drain the base batch
const next = rnd();
if (Array.isArray(t.state) && Array.isArray(next)) {
(t.state as unknown[]).splice(0, (t.state as unknown[]).length, ...next);
} else if (!Array.isArray(t.state) && typeof next === "object" && next !== null && !Array.isArray(next)) {
const s = t.state as Record<string, unknown>;
for (const k of Object.keys(s)) if (!(k in next)) delete s[k];
for (const [k, v] of Object.entries(next)) s[k] = v;
} else continue;
expect(apply(structuredClone(base), t.flush())).toEqual(t.target);
checked++;
}
// Most random pairs are shape-incompatible and skipped; this only guards
// against the loop silently checking nothing.
expect(checked).toBeGreaterThan(200);
});
});

285
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.ts vendored Normal file
View File

@@ -0,0 +1,285 @@
// ─────────────────────────────────────────────────────────────────────────────
// harness/delta — intent-recording change tracking over plain JSON.
//
// Depends on nothing else in the harness. Session storage, the runtime and the
// facet host consume it; keep the arrows pointing that way.
//
// State is plain TS. You mutate it normally. Ops come out carrying what you
// *did*, not what changed — `text += chunk` yields an append, not a 50 KB
// replacement; `arr.unshift(x)` yields one splice, not O(n) index writes.
// ─────────────────────────────────────────────────────────────────────────────
export type JsonValue = null | boolean | number | string | JsonValue[] | { [k: string]: JsonValue };
export type Seg = string | number;
export type Path = readonly Seg[];
/** Non-empty: `s`/`d`/`a`/`t` cannot target the root. Enforced by the type. */
export type NonEmptyPath = readonly [Seg, ...Seg[]];
/** A path inline on first use, an interned id thereafter. */
export type PathRef<P extends Path = Path> = P | number;
/**
* Tuples are the form — in memory, on the wire, on disk. There is no keyed
* variant and no codec between them.
*
* `r` is the ONLY op that replaces a whole value. It is an op rather than a
* frame kind for the same reason `a` and `t` exist: they are specialisations of
* `s` that are smaller and state intent. It also makes base detection a token
* comparison rather than a path inspection, which matters because recovery stops
* at the last base batch — a correctness boundary.
*
* Only `p` may target the root, and only because a tracked value can itself be
* an array. A `p` covering its entire target is normalised at RECORD time to `r`
* or `s`, so a root `p` is always a partial modification.
*
* The two-element forms reuse the previous op's path; arity disambiguates.
*/
export type Op =
| readonly ["r", JsonValue]
| readonly ["s", NonEmptyPath, JsonValue]
| readonly ["d", NonEmptyPath]
| readonly ["a", NonEmptyPath, string]
| readonly ["t", NonEmptyPath, number]
| readonly ["p", Path, number, number, JsonValue[]];
/**
* What crosses a boundary. Adds path interning and arity omission, and nothing
* else. `["r", value]` carries no path, so it encodes to itself — which is why
* `isBase` works on either vocabulary.
*/
export type WireOp =
| readonly ["r", JsonValue]
| readonly ["s", PathRef<NonEmptyPath>, JsonValue] | readonly ["s", JsonValue]
| readonly ["d", PathRef<NonEmptyPath>] | readonly ["d"]
| readonly ["a", PathRef<NonEmptyPath>, string] | readonly ["a", string]
| readonly ["t", PathRef<NonEmptyPath>, number] | readonly ["t", number]
| readonly ["p", PathRef, number, number, JsonValue[]]
| readonly ["p", number, number, JsonValue[]]
| readonly ["#", number, Path];
/**
* There is no frame type. What travels is `Op[]`.
*
* A **base batch** is one whose first op is `r`; the first batch of a stream is
* always one, and storage tags it "base" so recovery can stop there. Sequencing
* lives on the transport — a list element's `seq`, or an SSE `id:` — never in the
* payload. See `delta.md` §6.
*/
// ─── Safety ──────────────────────────────────────────────────────────────────
/**
* Path segments that reach the prototype chain. Rejected at record time AND at
* apply time, including through an interned id.
*
* Reserved as SEGMENTS, not as values: an object with a literal `__proto__` key
* replicates fine as a whole value; only a path walking through it is refused.
* That is a real restriction on what is mutable — see `delta.md` §7.1.
*/
export declare const RESERVED_SEGMENTS: ReadonlySet<string>;
export declare class UnsafePathError extends Error {
readonly segment: Seg;
}
/** Rejects reserved segments, symbol keys, and negative or non-integer indices. */
export declare function assertSafePath(path: Path): void;
/**
* Rejects anything `JSON.stringify` would not round-trip: Map, Set, Date, RegExp,
* typed arrays, class instances, functions, BigInt, cycles.
*
* `structuredClone` is NOT a JSON check — it clones a Map happily, and the op then
* carries `{}` while the producer keeps a real Map. Silent divergence.
*/
export declare function assertJsonValue(value: unknown): asserts value is JsonValue;
/**
* Validate an op's structure before applying it. A decoder must not trust tuple
* shape: an unvalidated applier spreads a string into an array given
* `["p",["xs"],0,0,"not-an-array"]`, and writes at the root given a string path.
*
* An unknown verb is an error, not a no-op — skipping it is how a newer producer's
* op vanishes and a replica drifts. This is the CVE-2025-55182 lesson
* (`delta.md` §7.3).
*/
export declare function assertValidOp(op: unknown): asserts op is Op;
/**
* The same, for the wire grammar, where ids and short forms are legal.
* `decode` uses this; `apply` uses `assertValidOp`. Validating a decoded op
* against the wire grammar is laxer than its type — a two-element `["s", value]`
* would pass and then be read as a path.
*/
export declare function assertValidWireOp(op: unknown): asserts op is WireOp;
// ─── Classification ─────────────────────────────────────────────────────────
//
// Ops are tuples, so classification is easy to hand-roll and easy to get wrong.
// `op[0] === "s"` is not enough — a set on any path is "s"; what makes a base is
// that it targets the ROOT. And because a tuple may omit the PathRef (arity
// disambiguates), `op[1]` is a PathRef or a value depending on length. These
// exist so nobody writes that comparison twice.
/** True for the `r` op. */
export declare function isReplace(op: Op): boolean;
/**
* True when this batch begins with `r` — a **base batch**. Exact rather than a
* heuristic, because flush guarantees `r` appears at index 0 or not at all.
* Storage tags such elements `"base"` so recovery can stop at the last one.
*/
export declare function isBase(ops: readonly (Op | WireOp)[]): boolean;
// ─── Codec ───────────────────────────────────────────────────────────────────
/**
* Path interning and arity omission. ONE PAIR PER STREAM — the id table spans a
* whole subscription or file, so a consumer joining later needs its own.
*
* The table resets on a base batch, because a reader replays from the last one
* with a fresh decoder and everything after it must be self-contained.
*/
export declare function encoder(): { encode(ops: readonly Op[]): WireOp[] };
export declare function decoder(): { decode(wire: readonly WireOp[]): Op[] };
// ─── Tracker ─────────────────────────────────────────────────────────────────
export interface TrackerOptions {
/**
* Cap on characters compared when deriving append/truncate from a string
* write. A window slide's overlap lies near the end, so a cap bounds cost
* without changing the result. Default 65536; 0 disables detection.
*/
maxOverlapScan?: number;
/** Merge compatible adjacent ops on flush. Default true. */
coalesce?: boolean;
}
export interface Tracker<T extends object> {
/**
* Mutate this. Plain TS: assignment, push, splice, delete, all of it.
*
* Assigning to it replaces the whole value: emits `["r", next]` and discards
* ops recorded before it. It must be a SETTER on the tracker — a plain
* property would be overwritten, swapping the proxy for a plain object and
* silently ending tracking.
*
* To force a base batch without changing the value, call `rebase()`.
*/
state: T;
/** The untracked backing object. Reads only — writing here emits nothing. */
readonly target: T;
/**
* Drain. Empty when nothing changed.
*
* The FIRST flush is always a base batch, carrying the value including any
* mutations made before it — a consumer starts with nothing, so a stream must
* open with something to apply deltas to.
*
* Ops that a later op makes unreachable are dropped first — a field written
* three times emits one op, and a parent replaced after its child emits one.
* Object key order is not preserved by that pass; see `delta.md` §5.
*/
flush(): Op[];
/** Discard without emitting. For a producer that just sent a base batch. */
discard(): void;
get dirty(): boolean;
}
export declare function track<T extends object>(root: T, options?: TrackerOptions): Tracker<T>;
// ─── Applier ─────────────────────────────────────────────────────────────────
/**
* Apply ops to a plain mutable object. Five verbs plus the dictionary entry, no
* domain knowledge, no library, no registry lookup.
*
* Returns the value, because a root set replaces it outright and cannot be done
* in place. Throws `PathError` when a path does not resolve, which a consumer
* treats exactly like a sequence gap: request a fresh base frame.
*
* Must not be pointed at a value produced by Immer, which deep-freezes.
*/
/**
* Fold ops into a value and return it — returned rather than mutated in place,
* because `r` replaces the value outright. Tolerates an `undefined` target,
* since a root op never reads it, so a consumer needs no base-batch branch.
*
* An `r` payload is **adopted, not copied**: the consumer owns the batch it was
* handed. Do not hand one batch to two in-process consumers, or their replicas
* will alias.
*/
export declare function apply<T>(target: T, ops: readonly Op[]): T;
export declare class PathError extends Error {
readonly path: Path;
}
// ─── Replica ─────────────────────────────────────────────────────────────────
export interface Replica<T> {
/** Current value. `undefined` until the first base frame. */
readonly value: T | undefined;
/**
* Feed one batch, with the sequence number the transport supplied. Returns
* "gap" when `seq` is not contiguous or a path failed to resolve; the caller
* then requests a fresh base batch.
*/
receive(ops: readonly Op[], seq: number): "ok" | "gap";
}
export declare function replica<T>(): Replica<T>;
// ─── Semantics ───────────────────────────────────────────────────────────────
//
// ROOT OPS
// `["s", [], v]` replaces the value; `["p", [], …]` splices it when the value
// is an array. The applier must special-case `path.length === 0` before
// resolving a parent — a prototype handling only a root set threw on the first
// randomised sequence that produced a root splice.
//
// STRINGS
// A write of `next` over `prev` derives ops from the longest suffix of `prev`
// that is a prefix of `next`:
// overlap === prev.length -> append(next.slice(overlap))
// overlap > 0 -> truncate(prev.length - overlap) + append(rest)
// overlap === 0 -> set(next)
// Always correct: the overlap is verified, not guessed. This is what makes a
// rolling window cheap — the case every effect-recording library degrades on.
//
// Use a native probe, not a hand-written KMP: `indexOf` a short head of `next`
// in `prev`, verify with `endsWith`. Measured 47x faster over 2000 slides on a
// 50 KB string, verified identical across 200.
//
// `t` counts UTF-16 code units, matching String.prototype.slice. Not bytes.
//
// ARRAYS
// Mutator methods are intercepted in the `get` trap, so intent is recorded
// before the engine performs index writes:
// push/unshift/pop/shift/splice -> one `p`
// sort/reverse/fill/copyWithin -> `s` of the whole array (rare)
// Direct index writes are honest `s` ops. `a.length = n` is translated —
// shrink to a splice, grow to a splice of nulls — never emitted as a path,
// because `length` is not a document location.
//
// TRANSACTIONS
// Several writes to one address in one commit must each be recorded against
// the previous one's result, not against pre-transaction state. The applier
// replays in order; anything else applies the second against a stale base.
//
// ALIASING
// Op payloads are structured-cloned at record time. Without this a later
// mutation retroactively changes an already-emitted op and a local replay
// aliases the source — an observed bug, not a hypothetical.
//
// UNDEFINED
// `x = undefined` normalises to `d`. JSON has no undefined, and a set with a
// missing value is indistinguishable from a lost value after a round trip.
//
// SCOPE
// JsonValue only. No Map, Set, Date, RegExp, class instances, getters, symbol
// keys, or cycles. That keeps this to five traps and one string algorithm, and
// makes structuredClone on payloads always safe.

604
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/02-scopes/implementation-handoff.md vendored Normal file
View File

@@ -0,0 +1,604 @@
# Scoped storage Step 1 — actionable implementation handoff
**Status: actionable, not implemented.**
**Prepared against:** `c4b0e35ab` (`dev`). Re-audit named source files if the implementation baseline changes materially.
This handoff implements storage scopes only. **Read this file before [`scopes.md`](scopes.md).** It supersedes conflicting details there, especially per-scope sequence spaces, operation-derived orphan detection, one-use scope IDs, raw-string scope arguments, sidecars ending in `.jsonl`, the proposed interned retirement tuple, and bundling JSONL delta/address encoding into the same change. The broader motivation and measurements in `scopes.md` remain design evidence; its signatures and implementation order are not current where this file differs.
Step 2—Chord-encoded JSONL values, address interning, compact tuple records, and their measurements—is a separate package. Do not begin Step 2 until Step 1 has been tested, reviewed, explicitly approved by the user, committed, and pushed.
## 0. Delivery protocol
This package has a mandatory user checkpoint:
1. Implement Step 1 only.
2. Run every required focused test, `npm run check`, and `./test.sh`.
3. Review the complete diff and report results. Delegated reviews, if used, run with provider `anthropic` and model `claude-fable-5`.
4. **Stop and wait for explicit user approval.**
5. After approval, commit only this package's files and push that commit.
6. Confirm the pushed commit before designing or implementing Step 2.
Do not combine the Step 1 and Step 2 commits. Do not commit or push before approval.
## 1. Mandatory reading
Read completely before editing:
1. `packages/agent/docs/harness.md` §§0.2–0.9, 1.1–1.7, 2.7–2.8, 3.7–3.8, 3.13, 4.4–4.8, 5.4, Part 7, Part 9.
2. `packages/agent/docs/values.md`.
3. `packages/agent/docs/mobile-handoff/README.md`.
4. [`scopes.md`](scopes.md), using this handoff where they conflict.
5. `../01-delta/delta.md` for vocabulary ownership only; Step 1 does not store Chord ops.
6. `../04-tool-output/harness-tools.md` §§7.1–7.7 for later-consumer context; do not implement its output redesign here.
7. `../05-assistant-output/message-update.md` §7 for later-consumer context; do not implement it here.
8. `packages/agent/src/harness/session/{types,values,commit,in-memory-storage-state,memory,session,index}.ts`.
9. Every file under `packages/agent/src/harness/session/jsonl/` and the `FileSystem` declaration/implementation in `packages/agent/src/harness/{types,env/nodejs}.ts`.
10. `packages/agent/src/harness/runtime/{progress,lane}.ts`, `runtime/drive/{terminal,response,deferred,tools,reconcile,structural,boundary}.ts`, and every caller of `operationCleanupWrites`. Inspect `tool-placement.ts` through the grep audit, but current source has no scoped delete there.
11. `packages/session-backends/sqlite-node/src/{index,sqlite/index}.ts`, `sqlite/{storage,repo}.ts`, `migrations/001_initial.sql`, and `session/{values,session-sequences}.ts`.
12. Shared storage/repository conformance and every focused test named in §9.
13. `scopes.variance.ts` for the covariant-address/invariant-write phantom mechanism only. Its current no-ID `EphemeralScope` and `retireScope(id: string)` signatures are superseded by §2 and must be updated in Slice A.
Do not use `dist/` as implementation input. Format 4 remains WIP; no R11 migration is part of this package.
## 2. Fixed scope contract
### 2.1 Runtime scope identity
A scope ID is a reusable logical lifetime name, not a globally one-use token:
```ts
export type SessionScope = { readonly kind: "session" };
export interface EphemeralScope {
readonly kind: "ephemeral";
readonly id: string;
}
export type Scope = SessionScope | EphemeralScope;
export function ephemeralScope(id: string): EphemeralScope;
```
`ephemeralScope(id)` requires a non-empty well-formed Unicode string and returns a frozen value. Equal IDs identify the same physical scope; object identity has no meaning. JSONL encodes IDs with `encodeURIComponent`. The encoded component must be at most 180 ASCII characters; reject longer IDs and lone-surrogate input. The physical filename always has a fixed `scope-` prefix, so encoded `.`/`..` values cannot become path segments. Do not directly interpolate unencoded caller input into a path.
No create/open-scope transaction exists. The first scoped value/list write creates physical state. A later `retireScope(scope)` ends everything in that scope through its assigned global sequence. A write under the same ID after retirement begins a new logical lifetime.
Harness operation scopes use `ephemeralScope(operationId)`. Generic application scopes may use stable names. Storage never parses a scope ID as an operation ID and never reads Harness operation state to decide lifetime.
### 2.2 Reuse and retirement boundary
Global sequence order defines reusable generations:
```text
seq 10 scoped write S
seq 20 scoped write S
seq 30 retireScope(S)
seq 40 scoped write S # new lifetime
```
For a given scope ID, only scoped records with `seq` greater than its latest committed retirement sequence are live. Memory and SQLite realize this by deleting current rows/maps at retirement and allowing later writes to recreate them. JSONL uses the retirement sequence as its replay boundary (§6).
Retirement is idempotent in the storage sense: retiring a scope with no current state is legal and advances the boundary. A later write still begins a new lifetime. Storage does not maintain an all-time used-ID registry or reject reuse.
A scoped write reaching the commit line after retirement is therefore a new-lifetime write. Harness invocation fencing and settlement drains must continue to ensure a late assistant/tool progress job cannot recreate operation output after terminal retirement. Add explicit tests for this boundary.
### 2.3 Explicit retirement is the sole authority
```ts
export function retireScope(scope: EphemeralScope): Write<SessionScope>;
```
Storage never infers orphanage from operation absence, namespace, current state, or a missing owner. The only valid lifetime boundary is a committed `retireScope` write. If an owner never retires its scope, that is an owner defect and the scope remains live.
Crash behavior:
- crash before the retirement transaction commits: the scope remains live and reopens;
- crash after retirement commits but before JSONL unlink: replay sees the retirement boundary, ignores pre-boundary sidecar records, and retries physical deletion;
- crash during a later reused lifetime: records after the latest retirement reopen normally.
Repository deletion separately removes every physical sidecar belonging to the deleted Session. It does not need operation semantics.
### 2.4 Address and write typing
Addresses carry a covariant scope tag; writes carry an invariant scope tag. Preserve value-type invariance independently:
```ts
interface Value<T, Sc extends Scope = SessionScope> { /* existing fields + scope */ }
interface ValueList<T, Sc extends Scope = SessionScope> { /* existing fields + scope */ }
declare function value<T>(namespace: string, key?: string): Value<T, SessionScope>;
declare function value<T>(namespace: string, key: string, scope: EphemeralScope): Value<T, EphemeralScope>;
declare function list<T>(namespace: string, key?: string): ValueList<T, SessionScope>;
declare function list<T>(namespace: string, key: string, scope: EphemeralScope): ValueList<T, EphemeralScope>;
```
A session-scoped address has no runtime scope ID. An ephemeral address carries its scope ID. Scope is part of physical identity: equal namespace/key addresses in session scope and ephemeral scope, or in two different ephemeral IDs, are distinct.
`setValue`, `deleteValue`, `appendList`, and `deleteList` preserve the address scope in `Write<Sc>`. Entry and usage writes are session-scoped. `retireScope` is session-scoped even though applying it deletes ephemeral state.
All writes in an ordinary transaction have one static scope. A session transaction may include session values, entries, usage, and one or more `retireScope` writes. It may not include a direct ephemeral set/delete/append. An ephemeral transaction may include value/list writes only. At runtime, every ephemeral write in one transaction must carry the same scope ID; two different IDs have the same TypeScript tag and require this assertion.
Make the minimum generic changes necessary to preserve this through `Write`, `CommittedWrite`, `Storage.commit`, Session mutation capabilities, `CommitDecision`, and lane commands. Follow the variance proof in `scopes.variance.ts`; do not make readers invariant or propagate unnecessary scope type parameters through read-only APIs.
### 2.5 One global sequence space
Memory, JSONL main records, JSONL sidecar records, and SQLite rows share one Session-global sequence space. Every admitted commit remains serialized and receives increasing sequences in admission order. Gaps remain legal.
- Memory continues using one `nextSeq`.
- SQLite continues allocating from `sessions.next_seq` inside the write transaction.
- JSONL main and sidecar appends share one commit queue and one resident `nextSeq`.
JSONL open computes the high-water mark from the main header and every complete record in every matching sidecar **before** deleting or ignoring retired physical files. A retired sidecar's sequences may then become gaps but can never be reused. Torn uncommitted final transactions do not advance the durable high-water mark.
No reservation record, per-scope sequence counter, or range allocator is permitted.
## 3. List tags
Step 1 adds only the mechanism required by later tracked-output recovery:
```ts
export interface ListElement<T> {
seq: number;
value: T;
tag?: string;
}
export interface ListReadOptions {
cursor?: ListCursor;
order?: "asc" | "desc";
limit?: number;
stopAtTag?: string;
}
export function appendList<T, Sc extends Scope>(
address: ValueList<T, Sc>,
element: NoInfer<T>,
tag?: string,
): ListAppendWrite<Sc>;
```
Tags are non-empty strings when present. Storage stores and returns the tag without interpreting the element.
Read semantics:
1. Apply address scope, exclusive cursor, and order.
2. Inspect at most the normalized `limit` elements.
3. Stop at and include the first element carrying `stopAtTag`.
4. Return that possibly shortened page.
`stopAtTag` never searches beyond the page limit. If absent from the page, the caller pages again from the last returned sequence. Do **not** add the draft `tag` filter; no current consumer needs it and combining filtering with the existing cursor shape is ambiguous.
SQLite may fetch the indexed `limit` rows and truncate at the first returned tag in TypeScript; storage still never parses payloads. Preserve the existing primary-key query plan with no temporary sort.
## 4. Step 1 durable payloads remain unchanged
This package changes lifetime and routing, not output representation:
- `pendingToolOutput` remains `Value<AgentToolResult<unknown>, EphemeralScope>`;
- `pendingAssistantFrames` remains `ValueList<AssistantMessageFrame, EphemeralScope>`;
- `operationToolMemo` remains a `JsonValue` scalar;
- JSONL transaction records remain readable keyed objects;
- assistant progress still appends one frame per accepted frame;
- tool checkpoints still replace whole snapshots;
- `message_update` and `tool_update` event shapes do not change;
- safe replay retains its current checkpoint behavior, including the known delete-before-replay bug owned by the later tool-output package.
Do not rename `pendingAssistantFrames`, introduce `pendingAssistantOutput`, store `WireOp[]`, add Chord trackers/codecs, redesign `AgentHarnessTool`, or change output cadence here.
## 5. Runtime ownership and cleanup
### 5.1 Addresses moved to the operation scope
Construct these addresses with `ephemeralScope(operationId)`:
- `operationToolMemo(operationId, invocationId, name)`;
- `pendingToolOutput(operationId, invocationId)`;
- `pendingAssistantFrames(operationId, responseEntryId)`.
All other current built-ins remain session-scoped. In particular, `pi.op.state`, `pi.op.meta`, `pi.op.tool_args`, `pi.op.preparation`, and `pi.pending.entry` remain in the main scope because their writes coordinate atomically with lane/operation state.
### 5.2 No cross-file cleanup transaction
Remove direct ephemeral deletes from transactions that also write session state:
- assistant response settlement;
- deferred-response supersession;
- tool outcome staging in `drive/tools.ts`;
- cancellation/recovery decisions;
- terminal per-address cleanup.
Current `drive/tool-placement.ts` has no ephemeral delete; do not invent a change there.
`operationCleanupWrites` stops scanning tool memos and tool outputs and stops constructing a state-directed assistant-frame delete. It keeps session-scoped operation/meta/args/preparation/pending-entry cleanup and adds exactly one `retireScope(ephemeralScope(operationId))` in the universal terminal suffix.
After removing the current staging/terminal scans, remove `operationToolMemoPrefix` and `pendingToolOutputPrefix`; current source has no other production consumers. Update the normative exported prefix-constructor count from five to three and update exact constructor tests rather than retaining dead inventory APIs.
An ephemeral-only delete remains legal. For example, the current safe-replay checkpoint delete remains because its transaction contains no session write; fixing its behavior belongs to the later tool-output package, not scoped storage or JSONL optimization.
### 5.3 Unreachable scoped residue
After a response settles, a deferred response is superseded, or a tool reaches `outcome_ready`, its former frames/checkpoints/memos may remain physically and logically addressable inside the still-active operation scope. Current scalar operation state no longer references them; no recovery or snapshot path scans the scope to infer authority. The terminal retirement deletes the whole scope.
Call this **unreachable scoped residue**, not orphanage. Update `harness.md` invariants that currently require an `outcome_ready` call to have no physically present memo/checkpoint. The replacement invariant is that settled child state never consumes or exposes residue and terminal retirement removes all scoped state atomically with operation completion.
Close and fault are controlled crashes and never retire the scope. Reopen restores scoped progress only through addresses derived from current authoritative operation state.
## 6. JSONL sidecars
### 6.1 Physical layout
The main session file remains unchanged except for committed scope-retirement records. Every main file owns one known sidecar directory at `${mainPath}.scopes`; this requires no dirname API or `FileSystem` extension. Create it lazily with `createDir(..., { recursive: true })`. Discover scopes with `listDir` on that exact path. A sidecar filename is exactly `scope-${encodeURIComponent(scope.id)}.scope`, subject to §2.1's 180-character encoded-component limit. The suffix does not end in `.jsonl`, and the sidecar directory itself is a directory, so repository listing never mistakes either for a Session.
A sidecar begins with this exact header:
```ts
interface JsonlScopeHeader {
v: 4;
kind: "scope_header";
sessionId: string;
scopeId: string;
storageVersion: 1;
}
```
Validate the exact session/scope identity before replay. Create the first header through a temp file inside `${mainPath}.scopes` plus atomic rename. Temporary filenames end in `.tmp` and are never discovered as sidecars. Use the existing `FileSystem` capability; do not add a dirname method, extend `JsonlStorageOptions`, or add Node-only filesystem access to agent core.
Step 1 sidecar writes retain the current object record spelling and carry/validate their scope ID in committed shapes. Each physical transaction remains one complete line or one array line. Sidecar transactions may contain only value/list writes for that exact scope.
### 6.2 Retirement record
The main log records an explicit committed write:
```ts
interface CommittedScopeRetireWrite {
kind: "scope";
op: "retire";
seq: number;
scopeId: string;
}
```
Its Step 1 JSON representation is the corresponding readable object. Do not use the draft `['!', addrId, seq]`: a scope is not an interned value/list address. Compact tuple spelling belongs to Step 2 and must be designed there.
Applying a committed retirement removes resident values/lists in that scope. After the complete main transaction is durable, JSONL synchronously serializes lifecycle cleanup on the same commit queue: close any owned sidecar resource, attempt unlink, then release the queue. Failure to unlink is non-fatal to the already-committed business transaction; the sidecar remains logically bounded by the retirement sequence and deletion is retried on reopen.
### 6.3 Replay and reusable IDs
Open performs these logical stages before admitting writes:
1. Read/repair the main file and parse complete main transactions.
2. Discover and validate matching sidecars; ignore unrelated files.
3. Parse complete sidecar transactions and repair torn final transactions.
4. Compute the global sequence high-water mark from the header and **all** complete main/sidecar records, including records later excluded by retirement.
5. Determine the latest retirement sequence per scope from main transactions.
6. Keep sidecar writes whose sequence is greater than that scope's latest retirement.
7. Merge retained main and sidecar transactions by global sequence and replay them in global order, preserving transaction boundaries and rejecting duplicate/non-monotonic sequences.
8. Advance to the precomputed high-water mark.
9. Remove sidecars with no records after their latest retirement; cleanup failure does not make retired contents live.
Main transactions have sequence gaps where sidecar transactions occurred. Sidecars have gaps where other scopes/main commits occurred. Both remain valid.
A sidecar containing records after a retirement is an active reused lifetime and must not be deleted. Serialize retirement cleanup against later writes so an old unlink cannot race and remove a newly reused sidecar.
Malformed interior records or identity mismatch in an active sidecar are storage corruption and fail open. A torn final transaction is discarded wholly. Temporary files left by interrupted atomic creation are cleanup artifacts and never become sidecars.
### 6.4 Legacy v3
Legacy v3 has no scopes. An ordinary Harness operation commits session-scoped acceptance before any progress write, so it upgrades first in the normal path. Still define and test the generic storage behavior when the first caller write is ephemeral: complete the existing v3-to-v4 main rewrite/usage adjustment under the global commit queue, then commit the scoped transaction without placing it in the main file. A crash before the sidecar append leaves no committed caller transaction and may reuse its uncommitted sequence.
### 6.5 Repository and fork behavior
- Repository `list()` ignores `${mainPath}.scopes` directories and their files through its existing non-file/`.jsonl` filtering.
- Repository `delete()` removes the main file through its current path, then removes `${mainPath}.scopes` recursively with `force: true`; a scope-directory cleanup failure rejects deletion after the main-file result is known, without inferring Harness state. Missing scope directories are legal.
- `JsonlStorage.close()` drains its admitted main/sidecar commits as today; `JsonlSessionRepo.close()` retains its current lifecycle behavior because repository ownership is a separate package. No progress scope is retired merely because a handle closes.
- Preserve the current fork allowlist exactly. Do not add generic application values or lists to either fork scope. Ensure source snapshots and backend fork readers ignore all ephemeral values/lists regardless of namespace; current destinations remain logically unchanged except that ephemeral state cannot leak.
- Fork destinations contain no sidecars or retirement state.
- WP08 may later replace fork materialization but must preserve this scope policy.
J1 compaction is excluded. When J1 eventually lands, it may omit a historical retirement record only after no physical sidecar can be made live by losing that boundary.
## 7. Memory backend
Extend `InMemoryStorageState` physical value/list identity with scope. Session scope and each ephemeral ID are distinct. Preserve one global `nextSeq` and current stats behavior.
Applying `CommittedScopeRetireWrite` deletes every scalar/list in that scope in the same synchronous application of the surrounding transaction. It does not delete entries or usage, which cannot be ephemeral. A later scoped write recreates state under the same ID.
Snapshots expose scope so fork code can reject ephemeral state, but `createForkSnapshot` keeps its current built-in allowlist and must not start copying generic session-scoped application values or lists. Instrumentation must still report exact committed write order, including retirement.
## 8. SQLite backend
Change WIP schema in place with no migration:
```sql
scalar_values(
session_id TEXT NOT NULL,
scope_id TEXT NOT NULL,
namespace TEXT NOT NULL,
key TEXT NOT NULL,
seq INTEGER NOT NULL,
value TEXT NOT NULL,
PRIMARY KEY(session_id, scope_id, namespace, key)
) WITHOUT ROWID;
list_values(
session_id TEXT NOT NULL,
scope_id TEXT NOT NULL,
namespace TEXT NOT NULL,
key TEXT NOT NULL,
seq INTEGER NOT NULL,
value TEXT NOT NULL,
tag TEXT,
PRIMARY KEY(session_id, scope_id, namespace, key, seq)
) WITHOUT ROWID;
```
Use `scope_id = ''` for session scope and the exact scope ID otherwise. Every point read, prefix scan, set/delete, append/delete, snapshot, and query-plan assertion includes scope identity.
Inside the same `BEGIN IMMEDIATE` transaction, a retirement write executes:
```sql
DELETE FROM scalar_values WHERE session_id = ? AND scope_id = ?;
DELETE FROM list_values WHERE session_id = ? AND scope_id = ?;
```
It still consumes its assigned global sequence through `sessions.next_seq`, even though no retired-scope tombstone row remains. A later write under that ID recreates rows naturally. Retirement rollback must restore both scoped rows and all sibling main writes.
Fork paths copy only `scope_id = ''`. Shared-container deletion remains Session-scoped and removes all values/lists regardless of scope along with the other Session rows.
## 9. Required tests
Port tests before or with each implementation slice. Use the repository Vitest binary from the owning package. Do not run the full Vitest suite directly.
### 9.1 Type and constructor tests
- Session addresses default to `SessionScope`.
- Ephemeral overloads retain `EphemeralScope` while preserving invariant `T`.
- Address scope is covariant for reads; write scope is invariant.
- Same-scope session transactions typecheck.
- Same-scope ephemeral transactions typecheck.
- Session writes plus `retireScope` typecheck.
- Direct session + ephemeral writes in one transaction fail with `@ts-expect-error`.
- Two ephemeral IDs evade static distinction but fail the runtime same-ID assertion.
- Scope object identity is irrelevant; equal IDs access the same state.
- Empty and overlong encoded IDs reject; separators/Unicode encode without path escape.
- Session and ephemeral addresses with equal namespace/key do not alias.
### 9.2 Shared backend conformance
Run identically on Memory, JSONL, and SQLite:
- ephemeral scalar set/get/replace/delete;
- ephemeral list append/page/whole-list-delete;
- independent scopes with equal namespace/key;
- session scope independent from equal ephemeral address;
- mixed session/ephemeral transaction rejection before mutation;
- two ephemeral IDs in one transaction rejection before mutation;
- retirement deletes every value/list in the exact scope and no other scope;
- retirement atomic with session sibling writes;
- retirement of absent state is legal;
- write after retirement recreates a new lifetime;
- repeated retirement advances the boundary without making later writes disappear;
- main, scope A, scope B, retirement, and reused-A commits receive globally increasing sequences in admission order;
- retired/deleted physical elements leave legal gaps and first post-reopen commit allocates above every complete prior record;
- list tags round-trip, page limits remain bounded, and ascending/descending `stopAtTag` includes the marker;
- stats ignore scoped values/lists and retirement;
- close drains admitted scoped commits and rejects later reads/writes.
### 9.3 JSONL focused tests
- first scoped write creates exactly one valid sidecar and no main value/list record;
- several writes in one scoped transaction remain one physical sidecar line;
- main and multiple sidecars replay in global sequence order;
- header `nextSeq` and all complete sidecar records contribute to reopen high-water;
- torn sidecar final transaction is discarded wholly and repaired;
- malformed interior active sidecar and header identity mismatch fail open;
- crash before retirement leaves scoped data live;
- crash after main retirement but before unlink leaves data logically absent and cleanup retries;
- unlink failure does not reject the already-committed terminal transaction;
- reuse after retirement replays only records after the latest retirement sequence;
- multiple retirement/reuse cycles choose the latest boundary;
- retirement cleanup is serialized so a later reuse cannot be deleted by an earlier unlink;
- ignored pre-boundary records still prevent sequence reuse;
- sidecars and temp files never appear in repository listing;
- repository deletion removes all sidecars;
- forks exclude generic application-owned ephemeral values/lists;
- legacy-v3 first session write and first scoped-write paths both produce valid v4 state;
- close/reopen preserves active operation progress and does not retire it.
### 9.4 SQLite focused tests
- schema and query helpers include `scope_id`; list rows preserve nullable tags;
- one transaction atomically deletes scoped scalar/list rows and writes terminal main state;
- forced failure after scope deletion rolls back deletion and sibling writes;
- reuse after retirement recreates rows under the same scope ID;
- `sessions.next_seq` advances for retirement;
- list paging plan still uses the scoped primary key with no temp b-tree;
- both per-file and shared-container layouts isolate Session and scope IDs;
- fork and Session deletion exclude/remove scoped rows correctly.
### 9.5 Harness integration
Update existing exact-write tests rather than weakening them:
- assistant/deferred settlement drops mixed-scope frame deletes;
- tool outcome staging drops mixed-scope checkpoint/memo deletes;
- unreachable residue is never consumed by snapshot/recovery after its owning child settles;
- safe-replay's current ephemeral-only delete remains ordered and fenced;
- every terminal leaf emits one retirement in the documented cleanup position;
- close/fault emits no retirement and reopen recovers current frames/checkpoint/memos;
- cancellation and unknown-outcome recovery retire only at terminal completion;
- late queued assistant/tool progress cannot commit after terminal retirement or reopen the scope;
- terminal cleanup no longer scans memo/output prefixes;
- every focused test and fixture found by the §12 old-delete/prefix greps is updated deliberately; do not rely on a historical numeric test count;
- every terminal path reached through `operationCleanupWrites` callers in `drive/{response,reconcile,structural,boundary}.ts` is covered, including each operation family and cancellation.
Likely focused files include:
```text
packages/agent/test/harness/values.test.ts
packages/agent/test/harness/{memory,jsonl}-storage*.test.ts
packages/agent/test/harness/{memory,jsonl}-session-repo*.test.ts
packages/agent/test/harness/runtime/drive-{terminal,retry-deferred,tools,reconcile,generation}.test.ts
packages/session-backends/sqlite-node/test/{storage,storage-conformance,repo,repo-conformance}.test.ts
```
Use compiler and grep guards to find additional affected tests rather than assuming this list is exhaustive.
## 10. Implementation slices
### Slice A — contract, types, Memory reference, list tags
Primary files:
```text
CREATE packages/agent/src/harness/session/scope.ts for `SessionScope`, `EphemeralScope`, `Scope`, `ephemeralScope`, scope phantoms/helpers, and runtime scope-ID validation
MODIFY packages/agent/src/harness/session/{types,values,commit,in-memory-storage-state,memory,session,index}.ts
MODIFY packages/agent/src/harness/session/testing/{conformance/storage,instrumented-storage,storage-decorator,gating-storage}.ts
MODIFY packages/agent/test/harness/{values,memory-conformance,memory-storage,storage-backed-session}.test.ts
MODIFY packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.variance.ts
```
1. Add scope values/address/write variance and reusable-ID semantics.
2. Add retirement committed write and one-scope transaction validation.
3. Add tag/`stopAtTag` behavior.
4. Implement Memory physical identity, retirement, global sequences, snapshots.
5. Pass type tests and Memory conformance before continuing.
### Slice B — SQLite
Primary files:
```text
MODIFY packages/session-backends/sqlite-node/src/sqlite/migrations/001_initial.sql
MODIFY packages/session-backends/sqlite-node/src/sqlite/{storage,repo}.ts
MODIFY packages/session-backends/sqlite-node/src/sqlite/session/{values,session-sequences}.ts
MODIFY packages/session-backends/sqlite-node/test/{storage,storage-conformance,repo,repo-conformance}.test.ts
```
1. Add scoped schema/query identity and tags.
2. Implement atomic `DELETE ... WHERE scope_id` retirement.
3. Preserve global sequence allocation, stats, branch indexes, shared-container behavior.
4. Exclude ephemeral rows from forks.
5. Pass SQLite focused/conformance tests before continuing.
### Slice C — JSONL sidecars
Primary files:
```text
CREATE packages/agent/src/harness/session/jsonl/scope-files.ts for fixed directory/file naming, exact header parsing/serialization, discovery, atomic first creation, torn-tail parsing/repair, and recursive cleanup helpers
MODIFY packages/agent/src/harness/session/jsonl/{types,codec,storage,repo,legacy-v3,index}.ts
MODIFY packages/agent/test/harness/jsonl-{storage,storage-conformance,session-repo,session-repo-conformance,v3-migration}.test.ts
```
1. Add sidecar naming/header/create/discovery.
2. Route scoped commits to sidecars while allocating globally.
3. Persist main retirement records and serialized unlink.
4. Implement multi-file high-water calculation and globally ordered replay with reusable boundaries.
5. Handle torn tails, corruption, v3, listing, delete, close, and forks.
6. Pass all JSONL tests before runtime migration.
### Slice D — Harness migration
Primary files:
```text
MODIFY packages/agent/src/harness/session/values.ts
MODIFY packages/agent/src/harness/runtime/{progress,lane}.ts to propagate scoped write generics through progress channels and lane commands
MODIFY packages/agent/src/harness/runtime/drive/{terminal,response,deferred,tools,reconcile,structural,boundary}.ts
INSPECT packages/agent/src/harness/runtime/drive/tool-placement.ts; current source needs no scoped cleanup change
MODIFY all focused runtime tests found by the §12 greps
```
1. Scope memo/tool/frame addresses by operation ID.
2. Remove mixed-scope deletes and cleanup scans.
3. Add terminal retirement in the universal suffix.
4. Preserve current output/replay/event behavior.
5. Prove late-write fencing and every terminal/close/recovery path.
### Slice E — documentation and final validation
Update current normative/reference docs:
```text
packages/agent/docs/harness.md
packages/agent/docs/values.md
packages/agent/docs/post-wp05-roadmap.md
packages/agent/docs/mobile-handoff/README.md
packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.md
packages/session-backends/sqlite-node/README.md
packages/agent/src/harness/telemetry.ts and generated `packages/agent/docs/telemetry-schema.md`: add the `scope` session-write item kind without implementing spans
```
Do not rewrite historical WP00–WP07 handoffs or released changelog sections. On `dev`, do not add changelog entries under the repository's main-only rule.
Run:
```bash
npm run check
./test.sh
```
Also run every modified focused test after each slice. Record full command results for the approval checkpoint.
## 11. Exclusions
Do not include:
- Chord `Op`/`WireOp` storage integration;
- JSONL address/path interning or compact tuple records;
- pending-output address renames;
- `ToolOutput`, tool API changes, replay seeding, memo/checkpoint atomicity, progress cadence, rate limiting, exec-env changes;
- compact `message_update`, assistant incremental reducer changes, or protocol replication changes;
- J1 main-log snapshot compaction;
- WP08 fork redesign beyond excluding ephemeral state from current forks;
- repository lifecycle contract changes;
- telemetry implementation beyond keeping the declared schema accurate for the new write kind;
- migrations or compatibility for older WIP format-4 SQLite schemas;
- application-level automatic scope ownership, operation-derived orphan inference, leases, TTLs, or background scope collection.
If implementation requires an excluded item, stop and revise this handoff before expanding scope.
## 12. Grep guards
Before final review, inspect every remaining match:
```text
operationToolMemoPrefix
pendingToolOutputPrefix
deleteList(pendingAssistantFrames
deleteValue(pendingToolOutput
scopeId
retireScope
stopAtTag
```
Expected outcomes:
- no session-scoped transaction directly deletes an ephemeral address;
- no terminal path omits retirement;
- no backend point read/write omits scope identity;
- no fork copies ephemeral state;
- no JSONL sidecar is treated as a Session file;
- no per-scope sequence counter exists;
- no operation-state/orphan inference exists in storage.
## 13. Exit condition
Step 1 is implementation-complete when:
- scope/address/write typing rejects mixed-scope transactions and runtime validation rejects two ephemeral IDs in one transaction;
- IDs are reusable through latest-retirement global sequence boundaries with no historical ID registry;
- Memory, JSONL, and SQLite share one global sequence space and pass common conformance;
- SQLite retirement deletes exact scoped rows atomically with terminal state;
- JSONL writes ephemeral values/lists only to sidecars, records retirement only in the main log, replays all files in global order, preserves high-water marks, and deletes retired sidecars without operation inference;
- list tags and bounded `stopAtTag` work identically across backends;
- current tool/assistant payload and event formats are unchanged;
- every terminal path retires once, while close/fault preserves active scope recovery;
- forks and repository listing/deletion handle sidecars correctly;
- focused tests, `npm run check`, and `./test.sh` pass;
- final review has no blocker;
- the complete diff and results have been presented to the user and implementation has stopped for approval.
Only after explicit approval may Step 1 be committed and pushed. Step 2 starts only after that push is confirmed.

574
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.md vendored Normal file
View File

@@ -0,0 +1,574 @@
# Session Storage: Scopes
> **Status:** Design evidence. The actionable Step 1 contract is [`implementation-handoff.md`](implementation-handoff.md), which supersedes this document on sequence allocation, reusable scope IDs, retirement authority/records, sidecar layout, implementation order, and the separation of JSONL encoding into Step 2.
>
> **Scope:** `packages/agent/src/harness/session/`, plus the call sites in
> `runtime/` listed in §10.
>
> **Depends on [01-delta](../01-delta/delta.md)** for the landed Chord op vocabulary that pending state is stored in (§2, §11) and for the `WireOp` form the record encoding carries (§12). The two
> are otherwise independent and compound: encoding shrinks every write, scopes
> move one class of write out of the main log entirely.
>
> Every claim here was checked against `runtime/lane.ts`, `runtime/progress.ts`
> and `runtime/drive/*.ts`. The type-level enforcement in §6 was verified with
> `tsc --strict`, including that mixed-scope commits are the *only* thing it
> rejects.
## 1. The problem
Operation state — pending tool output, pending assistant output — matters only
while the operation runs. It is crash-recovery scaffolding, not history.
The JSONL log is append-only, so it persists anyway. Worse, deleting it *adds*
records: `deleteValue` and `deleteList` write a line each, and those lines also
persist. Today only a full snapshot rewrite reclaims the space, and no in-place
compaction exists (`jsonl/storage.ts` has `createFromSnapshot`, used for fork).
Measured on 20 operations, each 400 assistant frames plus 2 tools at 60
checkpoints of a 50 KB window:
| | size |
| --- | --- |
| total JSONL | 93.89 MB |
| settled transcript entries — the part that is actually history | **0.06 MB** |
Three orders of magnitude of the file is scaffolding for finished operations.
**Two independent fixes, and they are not alternatives:**
- **Encoding** ([delta.md](../01-delta/delta.md)): value writes carry Chord ops rather than whole values, and addresses are interned. Takes the same workload to **5.32 MB in a single file**, with no change to atomicity. The op vocabulary and flush-time tracker have landed; storage integration has not.
- **Scopes** (this document): pending state leaves the main log entirely, so it is
never history to begin with.
Do the encoding first. Scopes are worth it because 5.32 MB of dead weight per 20
operations still accumulates, and there is no compaction pass to reclaim it.
## 2. Scope on the address
Scope has two halves that must not be confused: a **type-level tag** carrying no
data, and a **runtime scope id** that names the file.
```ts
export type SessionScope = { readonly kind: "session" };
export type EphemeralScope = { readonly kind: "ephemeral" };
export type Scope = SessionScope | EphemeralScope;
// Passing a scopeId is what makes an address ephemeral.
export function value<T>(namespace: string, key: string): Value<T, SessionScope>;
export function value<T>(namespace: string, key: string, scopeId: string): Value<T, EphemeralScope>;
export function list<T>(namespace: string, key: string): ValueList<T, SessionScope>;
export function list<T>(namespace: string, key: string, scopeId: string): ValueList<T, EphemeralScope>;
/** A main-log record retiring an ephemeral scope. See §5. */
export function retireScope(id: string): Write<SessionScope>;
```
The suffix is not decoration: `Session` already names the session interface in
`harness/session/types.ts`, and reusing it produces `TS2440`/`TS2484` plus an
ambiguous re-export from `session/index.ts`.
```ts
interface Value<T, Sc extends Scope = SessionScope> extends ScopedAddress<Sc> {
readonly namespace: string;
readonly key: string;
/** Present iff Sc is Ephemeral. Routes the write and names the sidecar. */
readonly scopeId?: string;
}
```
The id is a plain runtime string, because it is an operation id — generated at
runtime and unavailable to the type system. That is precisely why §6 can
distinguish *session from ephemeral* statically but cannot distinguish *two
different ephemeral scopes*: the tag is in the type, the id is not.
```ts
// Durable lists carry encoded batches; one encoder/decoder pair belongs to each value stream.
export const pendingToolOutput = (operationId: string, invocationId: string) =>
list<WireOp[]>("pi.pending.tool_output", `${operationId}:${invocationId}`, operationId);
export const pendingAssistantOutput = (operationId: string, entryId: string) =>
list<WireOp[]>("pi.pending.assistant_output", `${operationId}:${entryId}`, operationId);
export const operationToolMemo = (operationId: string, invocationId: string, name: string) =>
value<JsonValue>("pi.op.tool_memo", `${operationId}:${invocationId}:${name}`, operationId);
// unchanged — no scopeId, so Session by inference
export const laneStateValue = (lane: string) => value<DurableLaneState>("pi.lane.state", lane);
```
**Scope is the file.** A session-scoped write goes to the main log; an
ephemeral-scoped write goes to `<session>.<scopeId>.jsonl`.
Other backends need no sidecars at all: in-memory drops a `Map` keyed by scope id,
SQLite issues one `DELETE ... WHERE scope = ?`. Scope is meaningful to them only as
a lifetime hint. It is JSONL that needs the file split, and that is where the
constraints in §4 and §5 come from.
## 3. Which addresses are scoped, and why
The boundary is drawn by **write isolation**, not by lifetime.
Scoping everything under `pi.op.*` on the grounds that it dies with the operation
looks right and is wrong. `lane.ts` shows why: **every** operation commit bundles
`operationState` with `laneState`.
```ts
// lane.ts:405/431, 659/660, 749/750, 1071/1072 — the same shape each time
writes: [
...decision.writes,
setValue(operationStateValue(operationId), decision.operationState),
setValue(laneStateValue(this.name), durableLaneState(...)),
]
```
That pair is coordination state: lane state says the operation is at step N,
operation state holds the data for step N. Splitting them across files makes a
crash produce a lane that believes it is somewhere the data does not support.
The **bulk** values behave differently. `openProgress` commits exactly one write
(`runtime/progress.ts:51`), and `pendingToolOutput` / `pendingAssistantFrames`
appear in multi-write transactions only as *deletes*.
| | addresses |
| --- | --- |
| **ephemeral (sidecar)** | `pi.pending.tool_output`, `pi.pending.assistant_output`, `pi.op.tool_memo` |
| **session (main log)** | `pi.lane.state`, `pi.op.state`, `pi.op.meta`, `pi.result`, `pi.pending.entry`, `pi.branch.tip`, `pi.op.tool_args`, `pi.op.preparation` |
`operationToolMemo` is ephemeral so that the memo-plus-checkpoint bundling in
`harness-tools.md` §7.5 is a single-file transaction. `terminal.ts:34` already
treats `operationToolMemoPrefix` and `pendingToolOutputPrefix` as the same cleanup
class, so this matches how the code already thinks about them.
The size win is unaffected by leaving `operationState` behind: effectively all of
the 93.89 MB in §1 is pending tool and assistant output.
## 4. Two files are not atomic
Serialising on the Session line gives **ordering, not atomicity**. Two `write()`
calls to two descriptors, two fsyncs, no journal spanning them. A crash between
leaves the files disagreeing, and JSONL has nothing to repair it with.
So the design does not try to make cross-file transactions work. It ensures none
exist:
> **All writes in a transaction share one scope.**
That is achievable because it is already true of the code — see §5 for the audit.
Two consequences of *not* needing cross-file commits:
- **No write ordering between files is required.** The commit-record pattern
(sidecar first, fsync, then a main-log record acknowledging it) is unnecessary if
no transaction spans files. There is nothing to acknowledge.
- **No commit marker for sidecar-only writes.** One tempting design is a small
main-log record for `openProgress`, since nothing acknowledges its write. It is
not needed: that transaction is one write in one file and already atomic. Torn
tails are handled the same way as in the main log — every line is a complete op
or a whole batch, so a reader stops at the last intact one.
What remains is a **durability** choice, not a consistency one: if the sidecar is
fsynced less aggressively, recent checkpoints can be lost. That is the same bounded
loss the checkpoint interval already accepts, but it should be a deliberate policy
rather than an accident.
## 5. Audit: does the existing code satisfy the rule?
Every commit in `lane.ts` (12 sites) and every `writes:` producer in
`runtime/drive/*.ts` (32 sites) was read.
**Result: yes, with one adjustment.** No transaction writes to two files. The
spanning cases are all *deletes* of ephemeral state bundled with main-log writes:
- `response.ts:340` — the settle commit: `insertEntry`, `insertUsage`,
`setValue(branchTip)`, plus `deleteList(pendingAssistantFrames(...))`.
- `terminal.ts:50` `operationCleanupWrites` — bundles main-log deletes with
`toolMemos` and `toolOutputs` deletes, returned into the same array as
`operationResultValue` and `laneStateValue`.
- `tools.ts:230, 257` — `deleteValue(pendingToolOutput)` with main-log writes.
- `deferred.ts:157` — `deleteList(pendingAssistantFrames)`.
**The adjustment: retirement becomes a main-log record.**
```ts
retireScope(operationId): Write<SessionScope>
```
The log is the truth; the sidecar file is a cache. Recovery reads the retire record
and ignores — then removes — the sidecar. Unlinking eagerly after commit is a pure
optimisation; losing the unlink costs disk, never correctness.
This makes all four sites above single-scope, because the individual ephemeral
deletes disappear. It also *simplifies* `operationCleanupWrites`: the `scanValues`
calls for `operationToolMemoPrefix` and `pendingToolOutputPrefix` are replaced by
one `retireScope`.
And it makes the open-time sweep principled rather than heuristic. A sidecar is
dead iff the main log retired it or its operation is absent — no inference from
operation status.
### 5.1 Two ephemeral scopes in one transaction cannot arise
A lane holds `operation: { meta, state } | null` — **singular**. Every write set
uses one `operationId`, either `drive.operationId` or the newly created one in
`startOperation`, which asserts no operation is active. Retire and start are always
separate transactions: the settle commits at `lane.ts:427` and `response.ts:339`
set lane state to `null`.
Concurrency is across **lanes**, and each lane has at most one active operation, so
two ephemeral scopes never meet in a commit. The runtime assertion in §6 is defence
against a future change, not a live gap.
## 6. Static enforcement
> **`scopes.variance.ts` ships alongside this doc** and is the executable form of
> this section: `npx tsc --noEmit --strict --lib es2023 scopes.variance.ts`.
> Silence means the rule holds. Its three `@ts-expect-error` lines fail the
> compile if they ever start type-checking, so it catches the enforcement being
> weakened as well as broken.
Two phantom types, differing only in where `Sc` appears. This distinction is
load-bearing and is the single easiest thing to get wrong here.
```ts
declare const storedScopeType: unique symbol;
/** Addresses: COVARIANT — Sc only in return position. */
export interface ScopedAddress<Sc extends Scope> {
readonly [storedScopeType]?: () => Sc;
}
/** Writes: INVARIANT — Sc in both parameter and return position. */
export interface Scoped<Sc extends Scope> {
readonly [storedScopeType]?: (scope: Sc) => Sc;
}
export interface Value<T, Sc extends Scope = SessionScope>
extends StoredAddressBase, ScopedAddress<Sc> { … }
export interface ValueSetWrite<Sc extends Scope> extends Scoped<Sc> { … }
```
**Why they differ.** Reading an address is safe at any scope — `getValue` does not
care which file a value lives in, so `Value<T, SessionScope>` must be usable where
`Value<T, Scope>` is expected. Forming a transaction is *not* safe at any scope,
because two files are not atomic, so a commit must pin down exactly one.
Enforcement therefore belongs where transactions are formed, not where addresses
are read. `setValue<T, Sc>(address: Value<T, Sc>, …): Write<Sc>` is the bridge: it
infers the scope from a covariant address and stamps it onto an invariant write.
> **Getting this wrong is expensive.** Making addresses invariant too breaks every
> reader — `getValue`, `scanValues`, `readList` and everything downstream — and
> produced a 36-error cascade across storage backends that had done nothing wrong.
> The tempting workaround, making each reader generic over `Sc` to mean "any
> scope", propagates a type parameter through the whole storage stack to express
> something variance gives for free.
Verified with `tsc --noEmit --strict`. Reads pass at both scopes:
```ts
getValue(laneState); // Value<T, SessionScope>
getValue(pendingOutput); // Value<T, EphemeralScope>
```
Single-scope commits pass:
```ts
commit([setValue(laneState, a), setValue(operationState, b)]);
commit([setValue(toolMemo, m), setValue(pendingOutput, o)]);
commit([setValue(laneState, a), retireScope("op_1")]);
```
Mixed-scope commits fail, and are the *only* things that fail:
```ts
commit([setValue(laneState, a), setValue(pendingOutput, o)]); // ERROR
commit([setValue(pendingOutput, o), retireScope("op_1")]); // ERROR
```
### 6.1 What the type system cannot catch
Two different ephemeral scopes both type as `Write<EphemeralScope>`, because the id
is a runtime string (§2). A runtime assertion at commit covers it — compare
`scopeId` across all writes — and per §5.1 it should never fire.
## 7. Recovery and sweeping
On open, enumerate sidecars alongside the main log and load them. Their records
participate in replay exactly as main-log records do.
A sidecar is removed when the main log holds a `retireScope` record for it, or when
its operation is absent — a crash between commit and unlink. Torn-tail semantics
are unchanged: each line is self-contained, so a crash mid-write loses exactly the
trailing line.
## 8. Sequence numbers
Scoped writes get their own sequence space per scope. Nothing outside a scope
orders against its contents, and keeping them separate means `nextSeq` in the main
header does not account for numbers consumed by files that will be deleted.
Shared numbering would leave large gaps after several operations settled, and the
high-water mark would need separate persistence.
## 9. Measured effect
Same workload as §1:
| | size | of today |
| --- | --- | --- |
| today, single JSONL | 93.89 MB | — |
| ops + interned addresses, single JSONL | 5.32 MB | 5.7% |
| **plus scopes — main log** | **0.06 MB** | **0.06%** |
| — sidecars, retired on settle | 5.26 MB | peak 0.26 MB per operation |
The 5.7% figure is **rate-dependent and must not be quoted alone**: it holds at
2 KB of tool output per checkpoint and inverts above the cap, where a whole-value
replacement wins.
The scope result is **not** rate-dependent. The main log holds settled entries
regardless of encoding, and that is the number that governs session file growth.
## 10. What changes in the existing code
The compiler finds these for you: once addresses carry a scope and `Write<Sc>` is
invariant, every site that mixes scopes in one transaction fails to typecheck.
This list is what a full pass turned up, so you know when you are done.
**New, in `session/values.ts` and `session/types.ts`**
`SessionScope` / `EphemeralScope` / `Scope`; the two phantoms from §6;
`Value<T, Sc>` and `ValueList<T, Sc>`; `scopeId` on the address; the `value` /
`list` overloads; `retireScope`; scope-preserving `setValue` / `deleteValue` /
`appendList` / `deleteList`; `Write<Sc>` with entries, usage and retirement
session-only by construction.
**Generified**
`CommitDecision<TResult, Sc>`, `lane.command<TResult, Sc>`, `Storage.commit<Sc>`.
Give `Sc` a default of `SessionScope`: conditional types are not inference sites,
so without a default a mixed array falls back to the constraint `Scope` and every
call site fails. `settleOperation` and `continueOperation` stay **non**-generic —
see §5.
**Committed writes**
`CommittedScopeRetireWrite`, and `scopeId` carried through the committed shapes so
`JsonlStorage` can route on it.
**Call sites that must change** — each currently mixes scopes in one transaction:
| site | what it does today | what it becomes |
| --- | --- | --- |
| `drive/terminal.ts` `operationCleanupWrites` | enumerates tool memos and tool outputs via `scanValues`, deletes each alongside session deletes | one `retireScope(operationId)`; two of its four scans disappear |
| `drive/response.ts` (settle) | `deleteList(pendingAssistantOutput)` bundled with operation state | drop it — the sidecar is discarded at retire |
| `drive/deferred.ts` (superseded response) | same | same |
| `drive/tools.ts` (tool settle) | `deleteValue(pendingToolOutput)` plus memo deletes in a session write array | drop them |
| `drive/tool-placement.ts` | `deleteValue(pendingToolOutput)` | drop it |
The pattern is the same everywhere: **intra-operation cleanup of ephemeral state
is impossible**, because those commits also write operation and lane state. The
sidecar is discarded wholesale at retire. The cost is that a multi-turn operation
holds superseded pending output until settle — bounded by turns, and never
reaching the main log.
**Storage**
Sidecar routing in `JsonlStorage.commit` (assert one scope id per transaction,
route the append), unlink on a `scope` record **after** the commit, and an
open-time sweep that loads sidecars and removes any the main log already retired.
In-memory drops a `Map` keyed by scope id; SQLite issues one
`DELETE ... WHERE scope = ?` and treats a retire record as a no-op, because it has
no sidecar and the terminal transaction deletes the scoped values explicitly.
**Expected test churn**
Nine tests assert the old cleanup write set and will fail — `drive-terminal` (3),
`drive-retry-deferred` (2), `drive-tools`, `drive-reconcile`, `drive-generation`,
and one pinning reserved namespaces. They expect six or seven individual deletes
where the new code emits fewer plus one `retireScope`. That is the change landing,
not a regression.
## 11. List tags and stop conditions
Tracked state is stored as a list of op batches (`delta.md` §9). Recovery needs "the
batches since the last base batch" without unpacking every row, which lists cannot
express today: `ListReadOptions` has `cursor`, `order` and `limit` only.
`BranchScan` already solves the same problem for entries, and its vocabulary is
the one to copy — `stopAtType` / `stopAtId` for a terminator, plain field names
for a filter.
```ts
appendList<T, Sc>(address: ValueList<T, Sc>, element: T, tag?: string): ListAppendWrite<Sc>;
export interface ListElement<T> {
seq: number;
value: T;
tag?: string;
}
export interface ListReadOptions {
cursor?: ListCursor;
order?: "asc" | "desc";
limit?: number;
/** Include elements up to and including the first carrying this tag, then stop. */
stopAtTag?: string;
/** Return only elements carrying this tag. */
tag?: string;
}
```
**`stopAtTag` is a stop condition within a page, not a guarantee.** It can only end
a page earlier than `limit` would; it never overrides it. If the tagged element is
not in the page, the consumer sees no tagged element and pages again with the
cursor — the same loop `readAssistantFrames` already runs. There is no ordering
question between the two bounds and no "not found" error.
For that loop to work, `readList` must return the tag on each element. Otherwise a
consumer cannot tell "page ended at the tag" from "page ended at the limit"
without parsing the value, which is the thing the tag exists to avoid.
**The producer sets the tag, not storage.** For a tracked value the caller
already knows, because `isBase(ops)` is a token comparison on the first op
(`delta.md` §2):
```ts
const ops = tracker.flush();
if (ops.length === 0) return;
writes: [appendList(address, enc.encode(ops), isBase(ops) ? "base" : undefined)];
```
Storage must never inspect an element to derive a tag. That is what keeps op
batches opaque to the storage layer, and it is the same property that makes
`EntryScan.type` work: the discriminant is a stored column, not something derived
from the payload.
**The tag lives on the storage record**, beside `seq` and `value`:
```jsonc
["l",7,9,[["r",{…}]],"base"] // per §12.1
```
Storage must never parse an element to evaluate a predicate. That is what keeps
ops opaque to storage and the durable path free of domain knowledge
(`harness-tools.md` §7.7). It is also why `EntryScan.type` works: the discriminant
is a stored column, not something derived from the payload.
`order` stays `"asc" | "desc"` rather than `BranchScan`'s
`"newestFirst" | "oldestFirst"`. Branch order is semantic — a walk from a tip
through a tree. List order is over `seq`. Borrowing the branch words would imply a
traversal that is not happening.
Backends: SQLite gets a `tag` column and a real predicate. JSONL holds the list in
memory already, so it scans backwards and checks a field — no worse than today.
In-memory likewise.
The primitive generalises past frames: any list wanting "the tail since the last
checkpoint" gets it.
## 12. JSONL record encoding
Two dictionaries at two layers. They are the same trick and are independent: the
**address** dictionary is over `namespace` + `key` on the record; the **path**
dictionary is over paths *inside* a value's ops (`delta.md` §4).
### 12.1 Records
```
["@", addrId, namespace, key] address definition
["v", addrId, seq, wireOps] value write — WireOp[], delta.md §4
["l", addrId, seq, element, tag?] list append — element is WireOp[]
for a tracked value
["x", addrId, seq] delete (value or list)
["!", addrId, seq] retire an ephemeral scope
```
**Record verbs and op verbs are separate vocabularies read at different levels**,
but they must not look alike. `!` rather than `r` for retire: `r` is delta's
replace op, and a reader scanning a file should not have to remember which nesting
level they are at to know what a tuple means.
Retire takes an address id like everything else, defined by an `["@", id, …]`
whose namespace is the scope. That keeps one interning rule for the whole file
rather than a special case for one record.
Entries and usage rows keep their current keyed form: they are written once, never
repeat an address, and are read by machinery that has nothing to do with ops.
An address is defined on its **second** use, matching path interning — a
definition is pure overhead for an address written once, and a session has many of
those.
### 12.2 Worked example
Four writes across two addresses, before and after.
```jsonc
// today — 1250 bytes
{"kind":"value","op":"set","seq":7,"namespace":"pi.op.state","key":"01a04cf6-…","value":{"at":"starting","control":{…},"settings":{…},"latestAssistantEntryId":null}}
{"kind":"value","op":"set","seq":8,"namespace":"pi.lane.state","key":"main","value":{"currentOperationId":"01a04cf6-…",…}}
{"kind":"value","op":"set","seq":9,"namespace":"pi.op.state","key":"01a04cf6-…","value":{"at":"checkpoint",…}}
{"kind":"value","op":"set","seq":12,"namespace":"pi.op.state","key":"01a04cf6-…","value":{"at":"assistant.ready",…}}
```
```jsonc
// with both dictionaries — 547 bytes
["@",0,"pi.op.state","01a04cf6-…"]
["v",0,7,[["r",{"at":"starting","control":{…},"settings":{…},"latestAssistantEntryId":null}]]]
["@",1,"pi.lane.state","main"]
["v",1,8,[["r",{"currentOperationId":"01a04cf6-…",…}]]]
["v",0,9,[["#",0,["at"]],["s",0,"checkpoint"]]]
["v",0,12,[["s","assistant.ready"]]]
```
| line | before | after |
| --- | --- | --- |
| address definition | – | 61 |
| op.state -> `starting` (base batch) | 353 | 256 |
| address definition | – | 31 |
| lane.state (base batch) | 181 | 114 |
| op.state -> `checkpoint` | 355 | **48** |
| op.state -> `assistant.ready` | 361 | **37** |
| **total** | **1250** | **547** (44%) |
The shape matters more than the total. Base batches barely shrink — 353 to 256 is
envelope only, since the value ships whole either way. The transitions collapse
~8x because they carry one changed field instead of the whole state including the
never-changing `settings` block.
Both dictionaries appear on line 5: `0` in `["v",0,9,…]` is an **address** id,
while `["#",0,["at"]]` defines a **path** id inside the value's ops. Same trick,
different layers, separate tables. Line 6 then drops the path entirely — arity
omission, because it targets the same path as the previous op in that batch.
**Do not quote 44% as the file-level saving.** Dictionary entries are one-time, so
a real operation's ~11 op.state writes amortise them and the ratio improves; but
transcript entries are untouched, and they were 18 KB of a 67 KB tool-heavy run.
Applied there this hits the ~27 KB of value writes and would take the file down by
roughly a fifth, not by half.
### 12.3 Reading
A reader builds both tables while replaying, so a torn tail costs exactly the
trailing lines and no header needs rewriting. A snapshot rewrite starts fresh
tables and re-emits definitions naturally.
A numeric `addrId` for which no definition has been seen is a corrupt file, not a
recoverable state — unlike a missing path id, which cannot occur because path
definitions are inside the same record as their first use.
## 13. Open questions
- File-handle pressure across concurrent lanes, each with a live sidecar. Probably
fine, unmeasured.
- fsync policy for sidecars (§4) — matched to the main log, or relaxed given the
contents are bounded-loss scaffolding.
- Whether a long-running operation should rotate its sidecar. `rebase()`
(`delta.md` §3.2.3) bounds *recovery* length but not file size, so a command
running for hours still grows its sidecar without bound.
- Whether an in-place compaction pass for the main log is worth building anyway.
`createFromSnapshot` plus atomic replace is the mechanism; scopes reduce the need
but do not remove it, since superseded session-scoped values still accumulate.

65
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.variance.ts vendored Normal file
View File

@@ -0,0 +1,65 @@
/**
* Executable form of scopes.md §6.
*
* npx tsc --noEmit --strict --lib es2023 scopes.variance.ts
*
* Silence means the rule holds: reads accept either scope, single-scope commits
* pass, and mixed-scope commits are the ONLY thing rejected. The three
* `@ts-expect-error` lines fail the compile if they ever start type-checking,
* so this catches the enforcement being weakened as well as broken.
*
* The variance split is the part that is easy to get backwards. Making addresses
* invariant too breaks every reader — getValue, scanValues, readList and
* everything downstream — and the tempting fix, making each reader generic over
* Sc, propagates a type parameter through the whole storage stack to express
* what variance gives for free.
*/
declare const tag: unique symbol;
type SessionScope = { readonly kind: "session" };
type EphemeralScope = { readonly kind: "ephemeral" };
type Scope = SessionScope | EphemeralScope;
/** Addresses: COVARIANT — Sc only in return position. Readers take any scope. */
interface ScopedAddress<Sc extends Scope> { readonly [tag]?: () => Sc }
/** Writes: INVARIANT — Sc in both positions. Mixed commits cannot unify. */
interface Scoped<Sc extends Scope> { readonly [tag]?: (s: Sc) => Sc }
interface Value<T, Sc extends Scope = SessionScope> extends ScopedAddress<Sc> { k: "v" }
interface ValueList<T, Sc extends Scope = SessionScope> extends ScopedAddress<Sc> { k: "l" }
interface VSet<Sc extends Scope = Scope> extends Scoped<Sc> { kind: "value"; op: "set" }
interface VDel<Sc extends Scope = Scope> extends Scoped<Sc> { kind: "value"; op: "delete" }
interface LApp<Sc extends Scope = Scope> extends Scoped<Sc> { kind: "list"; op: "append" }
interface EntryWrite { kind: "entry" }
interface RetireWrite extends Scoped<SessionScope> { kind: "scope" }
type Write<Sc extends Scope = SessionScope> =
| VSet<Sc> | VDel<Sc> | LApp<Sc>
| (Sc extends SessionScope ? EntryWrite | RetireWrite : never);
declare function setValue<T, Sc extends Scope>(a: Value<T, Sc>, v: T): VSet<Sc>;
declare function deleteValue<T, Sc extends Scope>(a: Value<T, Sc>): VDel<Sc>;
declare function appendList<T, Sc extends Scope>(a: ValueList<T, Sc>, v: T): LApp<Sc>;
declare function retireScope(id: string): RetireWrite;
declare function insertEntry(e: unknown): EntryWrite;
declare function commit<Sc extends Scope = SessionScope>(w: readonly Write<Sc>[]): void;
declare function getValue<T>(a: Value<T, Scope>): T; // reader: any scope
const laneState = null as unknown as Value<string>; // session
const pendingOut = null as unknown as ValueList<string, EphemeralScope>; // ephemeral
const toolMemo = null as unknown as Value<string, EphemeralScope>;
// READS accept either scope — that is what covariance buys.
getValue(laneState);
getValue(toolMemo);
// Single-scope commits pass.
commit([setValue(laneState, "a")]);
commit([setValue(laneState, "a"), insertEntry({}), retireScope("op_1")]);
commit([appendList(pendingOut, "x"), setValue(toolMemo, "m")]);
// @ts-expect-error mixed scopes in one transaction
commit([setValue(laneState, "a"), appendList(pendingOut, "x")]);
// @ts-expect-error a retire is session-only, so it cannot join an ephemeral commit
commit([appendList(pendingOut, "x"), retireScope("op_1")]);
// @ts-expect-error entries are session-only
commit<EphemeralScope>([insertEntry({})]);

172
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/03-execenv/bash.after.ts vendored Normal file
View File

@@ -0,0 +1,172 @@
import { type Static, Type } from "typebox";
import type { Context } from "../context.ts";
import type { AgentHarnessTool } from "../types.ts";
import { getOrThrow } from "../types.ts";
import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult } from "../utils/truncate.ts";
import type { ExecutionToolContext } from "./tool-context.ts";
const MAX_TIMEOUT_SECONDS = 2_147_483_647 / 1000;
/**
* Coalescing cadence for live updates. The env owns capping and coalescing now
* (`executionenv.md` §3), so bash no longer has an update throttle or a
* checkpoint interval of its own.
*/
const BASH_UPDATE_INTERVAL_MS = 100;
const bashSchema = Type.Object({
command: Type.String({ description: "Bash command to execute" }),
timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })),
});
export type BashToolInput = Static<typeof bashSchema>;
export interface BashToolDetails {
truncation?: TruncationResult;
/** Set by the exec env, on the machine the command ran on. */
spillPath?: string;
}
export interface BashExecution {
command: string;
cwd: string;
env: Record<string, string>;
inheritEnv: boolean;
}
export type BashPrepare<TContext extends ExecutionToolContext = ExecutionToolContext> = (
execution: BashExecution,
toolContext: TContext,
context: Context,
) => void | Promise<void>;
export interface BashToolOptions<TContext extends ExecutionToolContext = ExecutionToolContext> {
commandPrefix?: string;
prepare?: BashPrepare<TContext>;
}
function validateTimeout(timeout: number | undefined): void {
if (timeout === undefined) return;
if (!Number.isFinite(timeout) || timeout <= 0) {
throw new Error("Invalid timeout: must be a finite number of seconds");
}
if (timeout > MAX_TIMEOUT_SECONDS) {
throw new Error(`Invalid timeout: maximum is ${MAX_TIMEOUT_SECONDS} seconds`);
}
}
export function createBashTool<TContext extends ExecutionToolContext = ExecutionToolContext>(
options?: BashToolOptions<TContext>,
): AgentHarnessTool<TContext, typeof bashSchema, BashToolDetails | undefined> {
return {
name: "bash",
label: "bash",
description: `Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`,
parameters: bashSchema,
async execute(_toolCallId, { command, timeout }, onUpdate, toolContext, _invocation, context) {
validateTimeout(timeout);
const { env } = toolContext;
const execution: BashExecution = {
command: options?.commandPrefix ? `${options.commandPrefix}\n${command}` : command,
cwd: env.cwd,
env: {},
inheritEnv: true,
};
await options?.prepare?.(execution, toolContext, context);
// The env caps, coalesces and spills at the source. bash keeps a mirror
// only to build the settled result; it never truncates and never chooses
// a spill path.
let view = "";
let truncation: TruncationResult | undefined;
let spillPath: string | undefined;
const publish = (checkpoint: boolean): void => {
onUpdate(
{
content: [{ type: "text" as const, text: view }],
details: { truncation, spillPath },
},
checkpoint ? { checkpoint: true } : undefined,
);
};
onUpdate({ content: [], details: undefined });
const result = await env.exec(
execution.command,
{
cwd: execution.cwd,
env: execution.env,
inheritEnv: execution.inheritEnv,
timeout,
capture: {
limits: { maxBytes: DEFAULT_MAX_BYTES, maxLines: DEFAULT_MAX_LINES, retain: "tail" },
intervalMs: BASH_UPDATE_INTERVAL_MS,
spill: true,
},
onUpdate: (update) => {
switch (update.kind) {
case "chunk":
view += update.text;
break;
case "snapshot":
view = update.output.text;
truncation = update.output.truncation;
spillPath = update.output.spillPath;
break;
case "counters":
truncation = update.truncation;
break;
}
// A snapshot is the durable-worthy state: it is the only kind that
// can express eviction, so it is where a crash loses least.
publish(update.kind === "snapshot");
},
},
context,
);
if (!result.ok) {
const error = result.error;
const status =
error.code === "timeout"
? `Command timed out after ${timeout} seconds`
: error.code === "aborted"
? "Command aborted"
: error.message;
throw new Error(view ? `${view}\n\n${status}` : status, { cause: error });
}
const captured = result.value.output;
view = captured.text;
truncation = captured.truncation;
spillPath = captured.spillPath;
let outputText = captured.text;
let details: BashToolDetails | undefined;
if (captured.truncation.truncated) {
details = { truncation: captured.truncation, spillPath };
const startLine = captured.truncation.totalLines - captured.truncation.outputLines + 1;
const endLine = captured.truncation.totalLines;
if (captured.truncation.lastLinePartial) {
// One line alone exceeded the byte cap. Line numbers would be
// misleading here, so report the line's own size instead — and only
// the env can measure it, since it holds the raw buffer.
const lastLineSize = formatSize(captured.lastLineBytes ?? captured.truncation.outputBytes);
outputText += `\n\n[Showing last ${formatSize(captured.truncation.outputBytes)} of line ${endLine} (line is ${lastLineSize}). Full output: ${spillPath}]`;
} else if (captured.truncation.truncatedBy === "lines") {
outputText += `\n\n[Showing lines ${startLine}-${endLine} of ${captured.truncation.totalLines}. Full output: ${spillPath}]`;
} else {
outputText += `\n\n[Showing lines ${startLine}-${endLine} of ${captured.truncation.totalLines} (${formatSize(DEFAULT_MAX_BYTES)} limit). Full output: ${spillPath}]`;
}
}
if (result.value.exitCode !== 0) {
throw new Error(
`${outputText ? `${outputText}\n\n` : ""}Command exited with code ${result.value.exitCode}`,
);
}
return { content: [{ type: "text", text: outputText || "(no output)" }], details };
},
};
}

148
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/03-execenv/execenv.md vendored Normal file
View File

@@ -0,0 +1,148 @@
# ExecutionEnv: bounded shell output
**Status:** implemented in production source. The prototype files beside this document are historical evidence; production lives in:
- `packages/agent/src/harness/utils/adaptive-publisher.ts`
- `packages/agent/src/harness/utils/output-capture.ts`
- `packages/agent/src/harness/env/nodejs.ts`
- `packages/agent/src/harness/tools/bash.ts`
The generic `ToolOutput` use of the same publisher remains in `04-tool-output`.
## 1. Problem
The old `Shell.exec()` accumulated complete strings inside `NodeExecutionEnv`:
```ts
stdout += chunk;
stderr += chunk;
```
Bash truncated only after those strings had already been built. `cat 1gb.txt` therefore materialized a gigabyte in the worker before any tool-level bound could help.
Spilling also belongs where bytes originate. If execution is remote, a spill on the worker is inaccessible to the model's `read` and `grep` tools, and creating it after transport would first send the complete gigabyte over the connection.
## 2. Boundary
`ExecutionEnv` now owns:
- a bounded head or tail view;
- complete byte and line totals;
- lazy source-local spill after the view first crosses its limits;
- persistent source-local spill with bounded write-stream backpressure;
- adaptive publication of the latest bounded state;
- a forced final publication before settlement.
It does not return or retain separate `stdout` and `stderr` values. Both pipes feed one arrival-ordered model-visible text view, matching bash and `ToolOutput`; preserving stream styling through tail eviction would require a segmented retained state that the Harness does not expose. Text is folded from updates, and `ShellExecResult` contains only exit and truncation/spill metadata.
Bash owns only command semantics and its model-visible footer. Its old rolling buffer, spill creation, 100 ms throttle, and full-output accumulation are gone. The existing two-second durable checkpoint request remains temporarily until `ToolOutput` owns durable cadence.
## 3. Contract
```ts
interface ShellOutputLimits {
maxBytes: number;
maxLines: number;
retain?: "head" | "tail";
}
interface ShellOutputCaptureOptions {
limits: ShellOutputLimits;
spill?: boolean;
}
type ShellOutputTruncation = Omit<TruncationResult, "content">;
interface ShellOutputMetadata {
truncation: ShellOutputTruncation;
spillPath?: string;
lastLineBytes?: number;
}
interface ShellOutputView extends ShellOutputMetadata {
text: string;
}
type ShellOutputUpdate =
| { kind: "replace"; output: ShellOutputView }
| { kind: "append"; text: string; metadata: ShellOutputMetadata }
| { kind: "slide"; drop: number; text: string; metadata: ShellOutputMetadata }
| { kind: "metadata"; metadata: ShellOutputMetadata };
interface ShellExecResult extends ShellOutputMetadata {
exitCode: number;
}
```
`drop` counts JavaScript string code units, matching `slice()`. Updates are ordered. A consumer applies them with `applyShellOutputUpdate()`.
A complete replacement establishes the initial state or recovers when no verified overlap exists. An append carries only a growing suffix. A slide drops a prefix and appends the new suffix. Metadata updates move totals or the spill path without resending text.
The compatibility `executeShellWithCapture()` helper still returns one bounded final view. Its `onChunk` callback receives initial, append, and slide text only; metadata and post-turnover replacements are not mislabeled as new bytes.
## 4. Adaptive publication
`AdaptivePublisher` keeps only the latest dirty state. Intermediate process writes never become a queue of output updates.
Policy is harness-global rather than per tool:
```ts
minIntervalMs = 100;
targetBytesPerSecond = 100 * 1024;
nextDelayMs = max(minIntervalMs, encodedUpdateBytes * 1000 / targetBytesPerSecond);
```
The first dirty state after idle is immediate. Writes received before the deadline collapse into the latest bounded state. One trailing timer guarantees eventual publication. Finalization bypasses the deadline once, still bounded by the retained cap.
The publisher commits its baseline before invoking the consumer. If a consumer applies an update and then throws, finalization cannot emit the same delta twice. The command fails with `callback_error`.
### Workload behavior
| workload | result |
| --- | --- |
| Finishes below cap | immediate initial state, small appends, forced final state |
| Below-cap trickle | isolated writes are immediate; sustained writes are at most 100 ms apart |
| Full-force output after cap | complete turnovers are cap-sized replacements spaced by their encoded size |
| Post-cap trickle | small verified `slide` updates remain responsive; the complete window is not resent |
| Burst then silence | one leading update and one trailing update |
| Error, timeout, or abort | latest bounded state is forced before the error settles |
At a 50 KB cap and 100 KB/s target, repeated complete turnovers settle near two updates per second. Small post-cap slides still use the 100 ms floor.
The rate bound is amortized. An immediate-after-idle update and forced terminal update may each create one cap-bounded burst.
## 5. Capture and spill
`OutputCapture` trims its decoded line-aware working buffer back to twice the byte cap when it exceeds four times the cap. Tail mode drops old text; head mode preserves the original prefix. This amortizes UTF-8 trimming instead of rescanning the retained window for every process chunk.
Spill creation is lazy. Before crossing, the source keeps at most the bounded prefix needed to create a complete archive. On crossing it:
1. pauses stdout and stderr while it creates the file inside the execution environment;
2. opens one persistent append stream with a bounded 8 MB high-water mark;
3. writes the preserved raw prefix and subsequent raw chunks in arrival order;
4. resumes immediately while the writer accepts data;
5. pauses again only when `write()` reports backpressure, then resumes on `drain`.
This avoids both an unbounded promise chain and one async file-open/append cycle per process chunk. A spill create or stream-write failure kills the child and fails execution rather than publishing lossy success.
The spill path is force-published as metadata when it becomes available. Spill writes are awaited before the final output flush.
Node streams remain raw for spill throughput and exact archival bytes. `OutputCapture` uses one streaming `TextDecoder`, so a read boundary cannot split a code point; invalid display control characters are removed only from bounded snapshots, not by scanning the complete raw stream. Line totals count a final unterminated line, and `lastLineBytes` remains exact even when one line exceeds the working buffer.
## 6. Remote execution
When worker and execution environment are colocated, updates are in-process and this publisher primarily bounds capture work. `ToolOutput` remains the downstream event/durability limiter.
When an execution environment is physically remote from its worker, the same bounded updates cross that transport. Full-force output cannot transfer the complete stream: intermediate writes collapse at the source, while the complete stream remains in the source-local spill.
Each real costly boundary gets its own publisher instance. A colocated deployment may bypass transport serialization; a custom tool that bypasses `ExecutionEnv` still passes through the future `ToolOutput` publisher.
## 7. Remaining work
- Move generic custom-tool composition, text retention, event publication, and durable cadence into `ToolOutput`.
- Replace whole `AgentToolResult` progress with Chord operations at that downstream boundary.
- Make memo and output checkpoint persistence one atomic transaction.
- Seed replay from durable output rather than deleting it.
- Put spills in an environment-owned session directory and sweep them after session lifetime plus a crash-retention floor.
- Decide explicit limits for images and structured details; text is bounded, those values are not yet.
- Define raw binary-output behavior. Current shell output remains lossy UTF-8 text.

137
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/03-execenv/nodejs-env.diff vendored Normal file
View File

@@ -0,0 +1,137 @@
diff --git a/packages/agent/src/harness/env/nodejs.ts b/packages/agent/src/harness/env/nodejs.ts
index 732dc84db..db9eb9a57 100644
--- a/packages/agent/src/harness/env/nodejs.ts
+++ b/packages/agent/src/harness/env/nodejs.ts
@@ -19,6 +19,7 @@ import { basename, isAbsolute, join, resolve } from "node:path";
import { createInterface } from "node:readline";
import { fileURLToPath } from "node:url";
import type { Context } from "../context.ts";
+import { OutputCapture } from "../utils/output-capture.ts";
import {
type ExecutionEnv,
ExecutionError,
@@ -28,7 +29,10 @@ import {
type FileKind,
ok,
type Result,
+ type ShellCapabilities,
type ShellExecOptions,
+ type ShellExecResult,
+ type ShellStream,
toError,
} from "../types.ts";
@@ -365,11 +369,15 @@ export class NodeExecutionEnv implements ExecutionEnv {
return ok(join(...parts));
}
+ capabilities(): ShellCapabilities {
+ return { capture: true, spill: true, liveUpdates: true };
+ }
+
async exec(
command: string,
options: ShellExecOptions | undefined,
context: Context,
- ): Promise<Result<{ stdout: string; stderr: string; exitCode: number }, ExecutionError>> {
+ ): Promise<Result<ShellExecResult, ExecutionError>> {
const signal = context.abortSignal;
if (signal?.aborted) return err(new ExecutionError("aborted", "aborted"));
const timeoutMsResult = resolveTimeoutMs(options?.timeout);
@@ -392,9 +400,28 @@ export class NodeExecutionEnv implements ExecutionEnv {
);
}
+ // Spill lives in the exec env, on the machine where the model's own `read`
+ // and `grep` run — the whole reason capping happens here rather than on the
+ // agent side (`executionenv.md` §1).
+ let spillHandle: string | undefined;
+ let spillChain: Promise<void> = Promise.resolve();
+ const wantSpill = options?.capture?.spill === true;
+
return await new Promise((resolvePromise) => {
- let stdout = "";
- let stderr = "";
+ const capture = new OutputCapture(options?.capture, options?.onUpdate);
+ const spillChunk = (chunk: string): Promise<void> => {
+ if (!wantSpill) return Promise.resolve();
+ spillChain = spillChain.then(async () => {
+ if (spillHandle === undefined) {
+ const created = await this.createTempFile({ prefix: "pi-output-", suffix: ".log" }, context);
+ if (!created.ok) return;
+ spillHandle = created.value;
+ capture.setSpillPath(spillHandle);
+ }
+ await this.appendFile(spillHandle, chunk, context);
+ });
+ return spillChain;
+ };
let settled = false;
let timedOut = false;
let callbackError: ExecutionError | undefined;
@@ -407,7 +434,7 @@ export class NodeExecutionEnv implements ExecutionEnv {
}
};
- const settle = (result: Result<{ stdout: string; stderr: string; exitCode: number }, ExecutionError>) => {
+ const settle = (result: Result<ShellExecResult, ExecutionError>) => {
if (timeoutId) clearTimeout(timeoutId);
if (signal) signal.removeEventListener("abort", onAbort);
if (child?.pid) this.activeChildPids.delete(child.pid);
@@ -458,31 +485,32 @@ export class NodeExecutionEnv implements ExecutionEnv {
}
}
+ // setEncoding("utf8") is what keeps a chunk boundary from splitting a
+ // codepoint; without it a multi-byte sequence straddling a read bakes in a
+ // replacement character (`executionenv.md` §8).
child.stdout?.setEncoding("utf8");
child.stderr?.setEncoding("utf8");
- child.stdout?.on("data", (chunk: string) => {
- stdout += chunk;
- try {
- options?.onStdout?.(chunk, context);
- } catch (error) {
- const cause = toError(error);
- callbackError = new ExecutionError("callback_error", cause.message, cause);
- onAbort();
- }
- });
- child.stderr?.on("data", (chunk: string) => {
- stderr += chunk;
+ const feed = (stream: ShellStream, chunk: string) => {
try {
- options?.onStderr?.(chunk, context);
+ capture.push(stream, chunk, context);
+ void spillChunk(chunk);
} catch (error) {
const cause = toError(error);
callbackError = new ExecutionError("callback_error", cause.message, cause);
onAbort();
}
- });
+ };
+ child.stdout?.on("data", (chunk: string) => feed("stdout", chunk));
+ child.stderr?.on("data", (chunk: string) => feed("stderr", chunk));
void waitForChildProcess(child).then(
- (code) => {
+ async (code) => {
+ // The spill chain is async and lazy: the file is created on the first
+ // chunk. Await it before snapshotting, or the returned CapturedOutput
+ // reports no spillPath for output that was in fact archived.
+ await spillChain;
+ capture.flush(context);
+ capture.dispose();
if (callbackError) {
settle(err(callbackError));
return;
@@ -495,7 +523,7 @@ export class NodeExecutionEnv implements ExecutionEnv {
settle(err(new ExecutionError("aborted", "aborted")));
return;
}
- settle(ok({ stdout, stderr, exitCode: code ?? 0 }));
+ settle(ok({ output: capture.snapshot(), exitCode: code ?? 0 }));
},
(error: Error) => settle(err(new ExecutionError("spawn_error", error.message, error))),
);

149
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/03-execenv/nodejs-env.test.diff vendored Normal file
View File

@@ -0,0 +1,149 @@
diff --git a/packages/agent/test/harness/nodejs-env.test.ts b/packages/agent/test/harness/nodejs-env.test.ts
index 361040495..ee28ef506 100644
--- a/packages/agent/test/harness/nodejs-env.test.ts
+++ b/packages/agent/test/harness/nodejs-env.test.ts
@@ -8,7 +8,6 @@ import { afterEach, describe, expect, it } from "vitest";
import { BACKGROUND_CONTEXT, withAbortSignal } from "../../src/harness/context.ts";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { FileError, getOrThrow } from "../../src/harness/types.ts";
-import { executeShellWithCapture } from "../../src/harness/utils/shell-output.ts";
import { createTempDir } from "./session-test-utils.ts";
const chmodRestorePaths: string[] = [];
@@ -303,7 +302,8 @@ describe("NodeExecutionEnv", () => {
BACKGROUND_CONTEXT,
),
);
- expect(result).toEqual({ stdout: `${await realpath(root)}:ok`, stderr: "", exitCode: 0 });
+ expect(result.output.text).toContain(`${await realpath(root)}:ok`.trim());
+ expect(result.exitCode).toBe(0);
});
it.each([
@@ -334,7 +334,7 @@ describe("NodeExecutionEnv", () => {
),
);
- expect(result.stdout).toBe(`${expectedSessionFile}|true|preserved`);
+ expect(result.output.text).toBe(`${expectedSessionFile}|true|preserved`);
},
);
@@ -358,7 +358,7 @@ describe("NodeExecutionEnv", () => {
),
);
- expect(result.stdout).toBe("::explicit");
+ expect(result.output.text).toBe("::explicit");
} finally {
if (previousInherited === undefined) delete process.env[inheritedKey];
else process.env[inheritedKey] = previousInherited;
@@ -396,7 +396,9 @@ describe("NodeExecutionEnv", () => {
await wslEnv.exec(`name='World'; echo "Hello, ${nameExpansion}!"`, undefined, BACKGROUND_CONTEXT),
);
- expect(result).toEqual({ stdout: "Hello, World!\n", stderr: "args:-s\n", exitCode: 0 });
+ expect(result.output.text).toContain("Hello, World!");
+ expect(result.output.text).toContain("args:-s");
+ expect(result.exitCode).toBe(0);
} finally {
process.chdir(originalCwd);
process.env.PATH = originalPath;
@@ -425,7 +427,7 @@ describe("NodeExecutionEnv", () => {
() => controller.abort(),
),
);
- expect(result.stdout).toContain("child-exiting");
+ expect(result.output.text).toContain("child-exiting");
} finally {
controller.abort();
cleanupDetachedChild(pidFile);
@@ -448,25 +450,30 @@ describe("NodeExecutionEnv", () => {
it("streams stdout and stderr chunks", async () => {
const root = createTempDir();
const env = new NodeExecutionEnv({ cwd: root });
- let stdout = "";
- let stderr = "";
+ // Streams are aggregated in arrival order now; `chunk` carries `stream` so a
+ // renderer can still colour stderr (`executionenv.md` §6).
+ let seen = "";
+ const streams: string[] = [];
const result = getOrThrow(
await env.exec(
"printf out; printf err >&2",
{
- onStdout: (chunk) => {
- stdout += chunk;
- },
- onStderr: (chunk) => {
- stderr += chunk;
+ onUpdate: (update) => {
+ if (update.kind !== "chunk") return;
+ seen += update.text;
+ streams.push(update.stream);
},
},
BACKGROUND_CONTEXT,
),
);
- expect(result).toEqual({ stdout: "out", stderr: "err", exitCode: 0 });
- expect(stdout).toBe("out");
- expect(stderr).toBe("err");
+ expect(result.exitCode).toBe(0);
+ expect(result.output.text).toContain("out");
+ expect(result.output.text).toContain("err");
+ expect(seen).toContain("out");
+ expect(seen).toContain("err");
+ expect(streams).toContain("stdout");
+ expect(streams).toContain("stderr");
});
it("reports a missing working directory before spawning", async () => {
@@ -484,7 +491,8 @@ describe("NodeExecutionEnv", () => {
const root = createTempDir();
const env = new NodeExecutionEnv({ cwd: root });
const result = getOrThrow(await env.exec("exit 7", undefined, BACKGROUND_CONTEXT));
- expect(result).toEqual({ stdout: "", stderr: "", exitCode: 7 });
+ expect(result.output.text).toContain("".trim());
+ expect(result.exitCode).toBe(7);
});
it("returns timeout errors for commands exceeding the timeout", async () => {
@@ -501,7 +509,7 @@ describe("NodeExecutionEnv", () => {
const result = await env.exec(
"printf out",
{
- onStdout: () => {
+ onUpdate: () => {
throw new Error("callback failed");
},
},
@@ -542,12 +550,23 @@ describe("NodeExecutionEnv", () => {
const root = createTempDir();
const env = new NodeExecutionEnv({ cwd: root });
const result = getOrThrow(
- await executeShellWithCapture(env, "yes line | head -n 15000", undefined, BACKGROUND_CONTEXT),
+ await env.exec(
+ "yes line | head -n 15000",
+ {
+ capture: {
+ limits: { maxBytes: 50 * 1024, maxLines: 2000, retain: "tail" },
+ spill: true,
+ },
+ },
+ BACKGROUND_CONTEXT,
+ ),
);
- expect(result.truncated).toBe(true);
- expect(result.fullOutputPath).toBeDefined();
- const fullOutput = getOrThrow(await env.readTextFile(result.fullOutputPath!, BACKGROUND_CONTEXT));
+ expect(result.output.truncation.truncated).toBe(true);
+ // The spill lives inside the exec env, on the machine the command ran on —
+ // which is where the model's own read and grep run (`executionenv.md` §1).
+ expect(result.output.spillPath).toBeDefined();
+ const fullOutput = getOrThrow(await env.readTextFile(result.output.spillPath!, BACKGROUND_CONTEXT));
expect(fullOutput.split("\n").length).toBeGreaterThan(10000);
- expect(result.output.length).toBeLessThan(fullOutput.length);
+ expect(result.output.text.length).toBeLessThan(fullOutput.length);
});
});

194
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/03-execenv/output-capture.ts vendored Normal file
View File

@@ -0,0 +1,194 @@
import type { Context } from "../context.ts";
import type { CaptureOptions, CapturedOutput, ShellOutputUpdate, ShellStream } from "../types.ts";
import {
DEFAULT_MAX_BYTES,
DEFAULT_MAX_LINES,
type TruncationResult,
truncateHead,
truncateTail,
utf8ByteLength,
} from "./truncate.ts";
/**
* Bounded capture for a Shell implementation.
*
* Owns the two-stage structure that `shell-output.ts` had inside `bash.ts`: a raw
* buffer trimmed to `maxBytes * 2` as a memory guard, and a retained view
* computed from it per snapshot. `truncateTail` is line-aware — it walks
* backwards line by line and takes a partial only when one line alone exceeds the
* cap — so eviction is not byte arithmetic and cannot be done incrementally.
*
* Emits the three update kinds from `executionenv.md` §4 and enforces the
* ordering invariant: once the cap is crossed, a `chunk` never follows without an
* intervening `snapshot`.
*/
export class OutputCapture {
readonly #maxBytes: number;
readonly #maxLines: number;
readonly #retain: "head" | "tail";
readonly #intervalMs: number;
readonly #onUpdate?: (update: ShellOutputUpdate, context: Context) => void;
/** Raw rolling buffer, held at twice the cap. */
#buffer = "";
/** Everything ever produced. */
#totalBytes = 0;
/** Newlines seen. A trailing partial line is counted separately — a command
* emitting one 60 KB line with no newline has produced one line, not zero. */
#newlines = 0;
#endsWithNewline = true;
/** True once the retained view has begun evicting (tail) or filled (head). */
#crossed = false;
/** Pending chunks for the current coalescing window. */
#pending: { stream: ShellStream; text: string }[] = [];
#lastFlush = 0;
#timer: ReturnType<typeof setTimeout> | undefined;
#spillPath: string | undefined;
constructor(
options: CaptureOptions | undefined,
onUpdate: ((update: ShellOutputUpdate, context: Context) => void) | undefined,
) {
this.#maxBytes = options?.limits.maxBytes ?? DEFAULT_MAX_BYTES;
this.#maxLines = options?.limits.maxLines ?? DEFAULT_MAX_LINES;
this.#retain = options?.limits.retain ?? "tail";
this.#intervalMs = options?.intervalMs ?? 0;
this.#onUpdate = onUpdate;
}
push(stream: ShellStream, text: string, context: Context): void {
if (text === "") return;
this.#totalBytes += utf8ByteLength(text);
this.#newlines += countNewlines(text);
this.#endsWithNewline = text.endsWith("\n");
this.#buffer += text;
// Memory guard. The retained view is computed from this, never equal to it.
const guard = this.#maxBytes * 2;
if (utf8ByteLength(this.#buffer) > guard) {
this.#buffer = trimToLastBytes(this.#buffer, guard);
}
this.#pending.push({ stream, text });
this.#schedule(context);
}
setSpillPath(path: string | undefined): void {
this.#spillPath = path;
}
/** The bounded view, recomputed from the raw buffer. */
snapshot(): CapturedOutput {
const truncation =
this.#retain === "head"
? truncateHead(this.#buffer, { maxBytes: this.#maxBytes, maxLines: this.#maxLines })
: truncateTail(this.#buffer, { maxBytes: this.#maxBytes, maxLines: this.#maxLines });
// The buffer already lost its front, so totals come from counters rather
// than from what survives.
const totalLines = this.#totalLines();
const truncated = this.#totalBytes > this.#maxBytes || totalLines > this.#maxLines;
return {
text: truncation.content,
truncation: {
...truncation,
lastLinePartial: truncation.lastLinePartial,
truncated,
truncatedBy: truncated ? (totalLines > this.#maxLines ? "lines" : "bytes") : null,
totalBytes: this.#totalBytes,
totalLines,
},
...(this.#spillPath === undefined ? {} : { spillPath: this.#spillPath }),
...(truncation.lastLinePartial ? { lastLineBytes: this.#lastLineBytes() } : {}),
};
}
#totalLines(): number {
return this.#newlines + (this.#endsWithNewline || this.#totalBytes === 0 ? 0 : 1);
}
/** Bytes in the final line of the raw buffer, for the single-oversized-line case. */
#lastLineBytes(): number {
const start = this.#buffer.lastIndexOf("\n");
return utf8ByteLength(start === -1 ? this.#buffer : this.#buffer.slice(start + 1));
}
#schedule(context: Context): void {
if (this.#onUpdate === undefined) return;
if (this.#intervalMs === 0) {
this.flush(context);
return;
}
if (this.#timer !== undefined) return;
const wait = Math.max(0, this.#lastFlush + this.#intervalMs - Date.now());
this.#timer = setTimeout(() => {
this.#timer = undefined;
this.flush(context);
}, wait);
}
/** Emit one update for everything buffered since the last flush. */
flush(context: Context): void {
if (this.#onUpdate === undefined) return;
this.#lastFlush = Date.now();
const pending = this.#pending;
this.#pending = [];
if (pending.length === 0) return;
const nowCrossed = this.#totalBytes > this.#maxBytes || this.#totalLines() > this.#maxLines;
if (this.#retain === "head") {
// Head never evicts. Below the cap the appended text is complete; above
// it, only totals move — resending maxBytes to change a number is the
// thing `counters` exists to avoid.
if (!nowCrossed) {
this.#onUpdate({ kind: "chunk", stream: pending[0]!.stream, text: joined(pending) }, context);
} else {
this.#onUpdate({ kind: "counters", truncation: this.snapshot().truncation }, context);
}
this.#crossed = nowCrossed;
return;
}
// Tail. Once the buffer is full every byte in evicts a byte out, so there is
// no cheap-trickle regime past the cap: chunks until full, snapshots after.
if (nowCrossed) {
this.#crossed = true;
this.#onUpdate({ kind: "snapshot", output: this.snapshot() }, context);
return;
}
if (this.#crossed) {
// Defensive: never let a chunk follow the crossing.
this.#onUpdate({ kind: "snapshot", output: this.snapshot() }, context);
return;
}
this.#onUpdate({ kind: "chunk", stream: pending[0]!.stream, text: joined(pending) }, context);
}
dispose(): void {
if (this.#timer !== undefined) clearTimeout(this.#timer);
this.#timer = undefined;
}
}
const joined = (pending: { text: string }[]): string => pending.map((p) => p.text).join("");
function countNewlines(text: string): number {
let count = 0;
for (let index = text.indexOf("\n"); index !== -1; index = text.indexOf("\n", index + 1)) count++;
return count;
}
/** Keep the last `bytes` UTF-8 bytes without splitting a codepoint. */
function trimToLastBytes(text: string, bytes: number): string {
if (utf8ByteLength(text) <= bytes) return text;
let low = 0;
let high = text.length;
while (low < high) {
const mid = Math.floor((low + high) / 2);
if (utf8ByteLength(text.slice(mid)) > bytes) low = mid + 1;
else high = mid;
}
return text.slice(low);
}
export type { TruncationResult };

211
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/bash-worked-example.md vendored Normal file
View File

@@ -0,0 +1,211 @@
# Worked Example: `bash` End to End
One tool through every layer. `execute` is simplified — timeout validation,
cancellation and exit-code branches are elided. Everything on the output path is
shown.
Layers: exec env → `ToolOutput` sink → harness events → durable storage → lane
state → facet → wire → consumer.
---
## 1. The tool
```ts
// packages/agent/src/harness/tools/bash.ts
export interface BashToolDetails { spillPath?: string; truncation?: ShellOutputTruncation }
export function createBashTool(): AgentHarnessTool<ExecutionToolContext, typeof bashSchema, BashToolDetails> {
return {
name: "bash",
parameters: bashSchema,
output: { retain: "tail", maxBytes: DEFAULT_MAX_BYTES, maxLines: DEFAULT_MAX_LINES },
async execute(_id, { command, timeout }, signal, out, context) {
const env = context.env;
let view: ShellOutputView | undefined;
const result = getOrThrow(await env.exec(command, {
cwd: env.cwd,
inheritEnv: true,
timeout,
capture: { limits: this.output, spill: true },
onUpdate: (u) => {
view = applyShellOutputUpdate(view, u);
if (u.kind === "append") out.write(u.text);
else out.replace(view.text);
out.details.truncation = view.truncation;
if (view.spillPath) out.details.spillPath = view.spillPath;
},
}, context));
if (result.spillPath) out.details.spillPath = result.spillPath;
if (result.truncation.truncated) out.write(`\n\n[${describe(result.truncation)}]`);
if (result.exitCode) throw new Error(`Command exited with code ${result.exitCode}`);
},
};
}
```
What is gone from today's implementation: the rolling `tailOutput` buffer,
`truncateTail`, `ensureFullOutputFile`, `createTempFile`,
`BASH_UPDATE_THROTTLE_MS`, `BASH_CHECKPOINT_INTERVAL_MS`, `updateDirty`,
`lastCheckpoint`, `scheduleOutputUpdate`, `emitOutputUpdate`,
`clearUpdateTimer`. About 40 lines of tool-local machinery, replaced by a capture
policy every tool gets.
`execute` returns `void`. `details` is one field, set once — and note the tool no
longer computes `truncation`, because the env owns the window and the tool no
longer knows what was dropped.
## 2. The exec env caps at the source
`env.exec` applies `ShellOutputLimits` where the bytes originate. For a sandbox host
that means `cat 1gb.txt` never ships 1 GB to the agent machine, and the spill
lands where the model's own `read` and `grep` run. See [`execenv.md`](../03-execenv/execenv.md).
The initial state is a bounded `replace`. Growth emits `append`; a moving tail emits `slide { drop, text }`; complete turnover falls back to a bounded `replace`; metadata moves totals and spill paths without resending text. The adaptive publisher keeps small slides responsive and spaces cap-sized turnovers according to their encoded size.
## 3. The sink
`ToolOutput` folds those into `ToolOutputState`:
```ts
{
content: [{ type: "text", text: "…the retained window…" }],
details: { spillPath: "/tmp/pi-session-x/bash-8f2.log" },
usage: undefined,
addedTools: undefined,
terminate: false,
truncation: { truncated: true, truncatedBy: "bytes", totalLines: 8123, totalBytes: 262144 },
}
```
`out.write(text)` appends to `content[0].text`; `out.replace(text)` assigns
the whole retained view when the env sends a `snapshot`, because an append cannot
express eviction. The tracker (`delta.md`) records intent either way: an append is
an `a`, and a window slide assigned wholesale becomes `t` + `a` via verified
overlap detection.
## 4. Ops
A 256 KB build over ~20 flushes, 50 KB window:
The first batch is a base batch — it begins with `r`, carrying the initial state. Everything
after is deltas. The repeated content path is interned on its second use, and
consecutive ops on it omit the id entirely:
```jsonc
[["r",{"content":[{"type":"text","text":""}],"details":{},"terminate":false,"truncation":{…}}]]
[["a",["content",0,"text"],"make: Entering directory …\n"]]
[["#",0,["content",0,"text"]],["a",0,"cc -c src/a.c …\n"]]
[["a",0,"cc -c src/b.c …\n"]]
…
[["s",["details","spillPath"],"/tmp/pi-session-x/bash-8f2.log"]]
…
[["t",0,4096],["a","cc -c src/z.c\n"],["s",["truncation","totalBytes"],262144]]
```
Details are one `s`, in one flush out of twenty. The `t` + `a` pair is the window slide. The current tracker recovers it by verified overlap; production remeasurement shows the generic path is already negligible, so no explicit append/truncate producer API is added ([decision](../01-delta/append-decision.md)).
## 5. Harness events
```ts
{ type: "tool_start", toolCallId: "call_7", toolName: "bash",
args: { command: "make -j8" } }
{ type: "tool_update", toolCallId: "call_7", ops: [ … ] }
{ type: "tool_end", toolCallId: "call_7", isError: false }
```
`tool_start` is identity only — no `caps`, no `initial`. The first batch is a base
batch, so the initial state arrives on the update channel, and the caps are
already inside it as `truncation.maxBytes` / `maxLines`.
`tool_end` carries no content and no details. Every byte already went out;
re-sending would duplicate any images.
## 6. Durable storage
`pendingToolOutput(operationId, "call_7")`, **ephemeral-scoped** to the operation,
so it lives in `<session>.op_….jsonl` and is retired on settle rather than
persisting in the main log forever. Retirement is a main-log `retireScope` record,
so it commits atomically with the settle writes; the unlink is a consequence of
replaying that record, not part of the transaction ([scopes.md §5](../02-scopes/scopes.md)).
A `list<WireOp[]>`, one encoded batch appended per flush, base batches tagged `"base"` so recovery reads backwards with `stopAtTag` and stops there ([scopes.md §11](../02-scopes/scopes.md#11-list-tags-and-stop-conditions)). The tracker emits structural ops; the producer calls `rebase()` periodically to write a capped root replacement and bound recovery replay. The durable interval is a sink policy. Shell capture controls only source-state and transport publication; memo, terminal, and recovery-base flushes are forced by `ToolOutput`.
Recovery seeds a fresh `ToolOutput` from that state — it does **not** delete it,
which is what `clearReplayCheckpoint` does today and is a bug
(`harness-tools.md` §7.4). If bash memoised "spilled to /tmp/…" and crashed, a
replay that discarded the state would create a second spill file and lose the
first.
The memo write and this checkpoint commit in the same transaction. Both are
ephemeral-scoped, so both land in the same sidecar — and the type system rejects a
commit that mixes scopes, so this cannot regress silently.
## 7. Lane state and the facet
```ts
export function reduceLaneSnapshot(view: LaneView, event: HarnessEvent): void {
switch (event.type) {
case "tool_start":
view.operation.tools.push({ id: event.toolCallId, name: event.toolName,
args: event.args, output: undefined });
return;
case "tool_update": {
const tool = view.operation.tools.find((t) => t.id === event.toolCallId);
if (tool === undefined) return; // host will send a base batch
tool.output = apply(tool.output, event.ops);
return;
}
case "tool_end": {
const i = view.operation.tools.findIndex(t => t.id === event.toolCallId);
if (i >= 0) view.operation.tools.splice(i, 1);
return;
}
}
}
```
Plain mutation, no Immer, no return value. An event for a tool the view has never
seen simply does nothing; the host sends a `replace`.
A lane facet runs this same function under the tracker, so the facet's own ops
fall out against **its** shape — which need not be `LaneView` and generally is
not. The facet never writes an op.
## 8. Wire and consumer
```jsonc
{ "seq": 0, "ops": [["r",{"transcript":[],"operation":null}]] }
{ "seq": 1, "ops": [ … ] }
```
One shape: a batch of ops. The first is a base batch — it begins with `r` —
and everything after is deltas. A gap, a reconnect, a provider
reload, or a fold that could not apply all take one path: send a fresh `replace`.
The consumer is `apply` — six verbs, no domain knowledge, no library, no tool
code, against a plain mutable object it owns.
## 9. What this run costs
256 KB of output, ~20 flushes, 50 KB window:
| | today | this design |
|---|---|---|
| durable writes | full `AgentToolResult` per checkpoint | structural ops plus explicit periodic capped base batches |
| durable location | main log, forever | sidecar, unlinked on settle |
| wire per flush | whole snapshot | one `truncate` + one `append` |
| details written | rebuilt whole, every flush | one `set`, once |
| truncation logic | in `bash.ts` | in the exec env, shared by all tools |
| spill location | `/tmp`, OS-cleaned | exec env, session-scoped |
The details row is the one worth dwelling on. Nothing about bash's details
changed — they were always small. What changed is that they stopped riding inside
a container replaced wholesale on every update, which is why no tool anywhere
currently bothers to mutate them incrementally.

462
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/harness-tools.md vendored Normal file
View File

@@ -0,0 +1,462 @@
# Tool Output and Progress
> **Scope:** harness-local. Nothing here depends on the facet system, RPC, or any
> presentation. Depends on [delta tracking](../01-delta/delta.md) (the landed Chord op vocabulary and tracker), [execution environments](../03-execenv/execenv.md) (where truncation happens), and [scoped storage](../02-scopes/scopes.md) (durability).
## 1. The problem
Three faults, one cause.
**Tools each implement their own truncation.** `bash.ts` owns a rolling buffer,
`truncateTail`, a spill file, an update throttle and a checkpoint interval.
`read.ts` has its own truncation. Every future tool producing bulk output
reimplements this, differently.
**Progress is a whole value.** `onUpdate(partialResult)` hands over a complete
`AgentToolResult` on every update, `tool_update` carries that whole result, and
`openToolProgress` persists it with `setValue`.
**`details: unknown` was believed to force this** — you cannot append to a value
whose shape you do not know. That premise is now false. A structural tracker
(`delta.md`) records ops against JSON without knowing the type, so details need
no special handling at all.
## 2. `ToolOutput`
The harness constructs a sink per invocation and passes it to `execute`. **Tools
return nothing**; the sink holds the result.
```ts
interface ToolOutput<TDetails extends JsonValue> {
/** Append to the text block. */
write(text: string): void;
/** Append an image block. Images are never windowed. */
image(image: ImageContent): void;
/** Replace the retained text wholesale. Chord still recovers a verified `t` + `a` slide when possible. */
replace(text: string): void;
/** Apply source-owned truncation totals and spill metadata without resending text. */
capture(metadata: ShellOutputMetadata): void;
/** The tool's own details object. Mutate it directly. */
readonly details: TDetails;
/** Accumulates. A subagent making several model calls adds to it. */
usage(usage: Usage): void;
/** Replaces. */
addTools(names: string[]): void;
/** Replaces — a tool that decides to terminate and then recovers can say so. */
terminate(value: boolean): void;
}
```
```ts
execute(
toolCallId: string,
params: Static<TParameters>,
signal: AbortSignal,
out: ToolOutput<TDetails>,
context: Context,
): Promise<void>;
```
`execute` returns `void`. Everything the tool produces flows through the sink —
including `usage` and `addTools`, which cannot be settle-time return values
because a replayed tool must be able to seed from durable state (§7.4).
**Failure is a thrown error.** `isError` is set by the harness when `execute`
rejects. A tool may throw anything, including errors from libraries it did not
write.
**`terminate` is orthogonal to how execution ended**, so a failing tool can ask
the loop to stop:
```ts
try { await thing(); } catch (error) { out.terminate(true); throw error; }
```
This closes a gap in the current implementation, where `executeToolCall`'s catch
hardcodes `isError: true` with no terminate, and `immediateError`'s terminate
parameter is only ever passed `true` by `applyBeforeToolDecision` for a hook
block.
### 2.1 Details are just an object
```ts
async execute(id, params, signal, out, context) {
out.details.total = 42;
out.details.passed += 1;
out.details.failures.push({ name, message });
out.details.current = undefined; // -> delete
}
```
Fully granular ops fall out, verified against the prototype:
```jsonc
["#",0,["details","passed"]]
["s",0,1]
["p",["details","failures"],0,0,[{…}]]
["d",["details","current"]]
```
No recipes, no mutation map, no `initialDetails` on `tool_start`, no Immer, and
no consumer ever runs tool code. `TDetails` remains the tool's own exported type
so a renderer can cast `call.details` to it — that is all it is for.
A survey of `packages/agent/src/harness/tools/` found **no tool mutating details
incrementally today**; only `bash.ts` writes them mid-stream and it rebuilds them
whole, because the container was replaced wholesale anyway. This design removes
that cause, and costs nothing if nobody takes it up.
**Caveat:** details are now unbounded. A tool pushing to `failures` in a loop
grows without limit, and unlike text there is no cap. Nothing shipped does this;
the door is open in a way it was not when details were a replace-only value.
### 2.2 Partial output survives failure
Today `executeToolCall` catches and returns `createErrorToolResult(message)`,
building a fresh result from the error string alone — a tool that streamed 8 KB
and then threw reports only the error.
With the sink, what was written is already the result. The harness appends the
error text as content and keeps the rest. The error text is model-visible
content, not presentation: the model needs to read why the call failed.
`abortedMessage` and `interruptedMessage` should be made consistent with this;
today they build replacement results via `syntheticMessage`, so a cancelled
long-running command loses its partial output too.
## 3. Content
Content is exactly **one text block followed by zero or more image blocks**. A
tool cannot interleave text between images — text written before and after an
image lands in the same block. Deliberate: truncation only ever touches a string,
and an image is never partially anything.
Images are **never windowed**. A byte or line cap over base64 payloads is
meaningless and `truncateTail` operates on text. `maxBytes` / `maxLines` govern
text only.
### 3.1 Retention mode
Declared on the tool definition:
```ts
output?: {
retain?: "head" | "tail"; // default "tail"
maxBytes?: number;
maxLines?: number;
}
```
**`head`** — append until the cap, then stop. Nothing is ever removed. Right for
output meaningful from the start: file reads, listings, greps.
**`tail`** — a rolling window. Right for anything whose interesting part is at
the end: builds, test runs.
Head+tail is **not** offered. `truncate.ts` exports `truncateHead` and
`truncateTail`; neither combines them and none is being added.
### 3.2 The tool does not truncate — the exec env does
For output originating in the execution environment, capping, coalescing and
spilling happen **at the source**. See [`execenv.md`](../03-execenv/execenv.md). The tool passes its
`ShellOutputLimits` into `env.exec` and pipes the resulting updates into the sink.
This is not a convenience. Reading a 1 GB file on a sandbox host must not ship
1 GB to the agent machine to be capped there, and the spill file must land where
the model's own `read` and `grep` run.
For output originating on the agent machine (subagents, in-process work),
`ToolOutput` applies the same logic locally. Same code, different location.
Note this reverses an earlier decision that there is no spill in the sink and a
temp-file path is tool-specific. The exec-env argument — model reachability
across a machine boundary — is what changed it.
## 4. Tool output state
`tool_update` carries `Op[]` (`delta.md` §6) targeting the invocation's
`ToolOutputState`:
```ts
interface ToolOutputState {
content: (TextContent | ImageContent)[];
details: JsonValue;
usage?: Usage;
addedTools?: string[];
terminate: boolean;
truncation: ShellOutputTruncation; // totals over everything ever written, without duplicate text
}
```
Ops rather than a typed variant union, for one decisive reason: **only ops give
details granularity without the harness knowing `TDetails`**. A typed union would
need per-tool recipes, which is the machinery §2.1 deletes.
Text still gets delta treatment, because the sink applies the cap *before*
mutating, so a rolling window produces `truncate` + `append` on
`content[0].text` rather than a whole-value set.
Interned ops also measure **smaller than a typed frame vocabulary** on every
workload, and 10x smaller on details, because a frame repeats `toolCallId` where
an interned op carries one integer. See `delta.md` §4.1.
There is no `drop` event. The sink knows what it evicted and expresses it as
`truncate`; a consumer needs no separate signal and derives nothing.
## 5. Harness events
```ts
| { type: "tool_start";
runId; turnId; toolCallId; toolName;
args: unknown }
| { type: "tool_update";
runId; turnId; toolCallId;
ops: Op[] } // a base batch begins with `r`
| { type: "tool_end";
runId; turnId; toolCallId;
isError: boolean }
```
**`tool_start` carries identity only.** An earlier draft added `caps` and
`initial`; both were redundant and are gone. The first batch is always a base batch
(`delta.md` §6), so the initial state arrives on the update channel — carrying it
twice means two ways to establish the base, which can disagree. And `caps` are
already inside the state: `ToolOutputState.truncation` carries `maxBytes` and
`maxLines`, which is what a renderer needs to say "50 KB limit". The draft itself
conceded a consumer "no longer has to" fold identically, since the producer's ops
encode eviction — that was the last reason for `caps` to travel, and it does not
hold.
`tool_update` carries ops. A base batch travels the same channel as a delta,
because a replacement is itself an op (`delta.md` §2) — there is no second shape. `message_update` has the identical shape
(`message-update.md` §5.1); a consumer folds tool output and assistant output with
one code path.
**`tool_end` carries no content, no details, no usage, no terminate.** Every byte
already went out. Re-sending would duplicate each base64 image at the moment
nothing new has happened.
> **The fold is the result.** Nothing a consumer folded is ever re-sent to
> confirm it.
This dissolves an earlier open question about settle-time truncation disagreeing
with the running fold. There is no separate settle-time truncation: the sink's
window *is* the truncation. Anything a tool wants to add at the end — bash's
`[Showing lines 8000-8123 of 8123]` footer — is `out.write(footer)`, one more
append.
The harness still assembles `AgentToolResult` in process for the model, and
`createToolResultMessage` still writes the settled `ToolResultMessage` to the
transcript with `content`, `details`, `usage`, `addedToolNames`, `isError`.
Neither is a wire event.
## 6. Lane reduction
```ts
export function reduceLaneSnapshot(view: LaneView, event: HarnessEvent): void;
```
Plain mutation on a plain object. No `Draft`, no `produce`, no Immer, and **no
`Rebase` return value** — a fold that cannot apply an event leaves state alone,
and the host sends a `replace`. (An earlier draft had `void | Rebase`; under
Immer that throws, and without Immer there is nothing to return to. Note
`reducer.ts:4` currently defines `LaneSnapshotReduction = LaneSnapshot | { rebase: true }`,
so this is a real change to existing code, and `navigation_end` is the event that
drives it.)
The event is the only input besides the view. No registry, no `resolve`, no tool
code — so a tool rewritten between crash and resume cannot make a persisted
stream unreadable.
When the harness is wrapped by a facet, the same mutation runs under the tracker
from `delta.md` and ops fall out. The harness itself does not know this.
## 7. Durability
### 7.1 What exists today
`pendingToolOutput(operationId, invocationId)` is a
`value<AgentToolResult<unknown>>`, and — importantly — `progress.write(partial)`
fires **only** when `options?.checkpoint === true` (`drive/tools.ts:325`). Not on
every update. bash checkpoints every 2 s with a `JSON.stringify` dedupe; every
other tool never checkpoints at all.
It is also **not a progress buffer**. It is the interruption checkpoint. On
resume, if the tool is not replay-safe, `readCheckpoint` turns it into a real
`ToolResultMessage` for the model: `[...checkpoint.content, INTERRUPTION_MARKER]`
plus `details` and `usage`.
Note the implication that misled an earlier draft: because the checkpoint must
*be* the current state, it was read as needing `value` semantics. It does not. It
needs to be *derivable*, and folding encoded batches from the last base batch derives it — which is the point of tagging base batches (§7.3). `checkpoint: true`
requests a durable write, not a replacement.
`pendingAssistantFrames` is a `list<AssistantMessageFrame>` appended per frame,
`deleteList` on settle in `response.ts:345`, `deferred.ts:157`, `terminal.ts:47`.
Note `operationCleanupWrites` (`terminal.ts:26`) does four `scanValues` calls to
enumerate what to delete at settle. Under scopes, the two covering
`operationToolMemoPrefix` and `pendingToolOutputPrefix` are replaced by a single
`retireScope(operationId)`.
### 7.2 Renaming
`pendingAssistantFrames` → **`pendingAssistantOutput`**, matching
`pendingToolOutput`. Frames stop being the durable unit; both addresses now hold
tracked state written as ops or snapshots.
### 7.3 What to write, and when
Both addresses are **ephemeral-scoped** ([scopes.md](../02-scopes/scopes.md)), so they live in a
sidecar retired on settle rather than in the main log forever.
Two independent wins, in order of importance:
- **Encoding.** Ops instead of whole values, plus address interning: 93.89 MB to
5.32 MB in a single file, no change to atomicity. Do this first.
- **Scopes.** Pending state leaves the main log entirely: 5.32 MB to 0.06 MB
surviving.
**Both are `list<WireOp[]>`**, not values. The sink owns one stateful Chord encoder/decoder pair per durable value stream. One encoded batch is appended per flush, and a logical batch whose first op is `r` is tagged `"base"` on the storage record.
Recovery reads backwards with `stopAtTag: "base"` and applies forward
([delta.md §9](../01-delta/delta.md#9-durable-form), [scopes.md §11](../02-scopes/scopes.md#11-list-tags-and-stop-conditions)).
Writing one flush:
```ts
const ops = out.flush();
if (ops.length === 0) return;
const wire = enc.encode(ops);
writes: [appendList(address, wire, isBase(ops) ? "base" : undefined)];
```
`isBase` comes from Chord. It checks for the root replacement op `r`; ordinary nested sets use `s`. Classification lives beside the vocabulary so the comparison is written once.
The landed tracker emits structural ops unconditionally. There is no serialized-size comparison or adaptive replacement heuristic. A producer requests a base batch explicitly with `rebase()`; the output sink's cap bounds that replacement, while periodic rebasing bounds recovery work. Production remeasurement rejects a text-specific append/truncate API: the generic path measured 2.43–2.46 µs per 50 KB rolling-window flush locally, below surrounding costs. Keep ordinary tracked string mutation; see the [decision record](../01-delta/append-decision.md).
**Checkpointing is not deleted.** `BASH_CHECKPOINT_INTERVAL_MS` remains only as an interim compatibility mechanism. The generic sink owns durable frequency because Shell cannot price a storage write or enforce memo/output atomicity. Forced memo, terminal, and recovery-base writes bypass ordinary pacing while remaining cap-bounded.
### 7.4 Replay must seed, not discard
`clearReplayCheckpoint` currently writes `deleteValue(pendingToolOutput(...))`
before re-executing a replay-safe tool (`drive/tools.ts:257`). **This is a bug.**
Replay-safe means the tool is re-executed, but memos exist precisely so it does
*not* redo work it already did — and skipped work emits nothing. Any output for
memoised work is lost today.
The fix: seed a fresh `ToolOutput` from the durable state, then re-execute. The
tool appends to a sink that already holds what it produced before the crash.
This is also why nothing may be settle-only. `usage` and `addTools` must survive
a seed, so they flow through the sink like everything else.
### 7.5 The memo invariant
> A tool's memo write and its output checkpoint must commit in the same
> transaction.
Otherwise a tool does work, sets a memo, crashes before the next checkpoint, and
on replay skips the work while the seeded output has no record of it.
**This does not hold today.** `setMemo` (`drive/tools.ts:112`) and `openProgress`
(`runtime/progress.ts:44`) are two separate `lane.command` calls, hence two
transactions. Making it hold means bundling the checkpoint into the memo's commit:
```ts
setMemo(name, value) {
validateMemoName(name);
if (!active) return Promise.reject(ended());
return lane.command<void>((state) => {
if (!ownsEffect(state)) return { kind: "reject", error: ended() };
const memo = operationToolMemo(drive.operationId, call.resultEntryId, name);
return {
kind: "commit",
writes: [
value === undefined ? deleteValue(memo) : setValue(memo, value),
setValue(pendingToolOutput(drive.operationId, call.resultEntryId), out.snapshot()),
],
next: state,
materialize: () => undefined,
};
}, drive.context);
}
```
Both addresses are **ephemeral-scoped**, so this is a single-file transaction and
statically enforced as one ([scopes.md §3 and §6](../02-scopes/scopes.md)). `operationToolMemo` is
placed in that scope precisely for this. Ordering on the Session line would not be
enough — two file writes are not atomic. The periodic checkpoint stays as it
is — best-effort, for the interruption path; this forces one where correctness
requires it.
The invariant holds because the tool does X, writes X's output to the sink, *then*
calls `setMemo("did X")` — so the sink's state at commit time already contains X's
output.
**Ordering alone does not work**, in case it looks tempting:
- memo first, checkpoint second → crash between → replay skips X and the seeded
output lacks it → silent loss;
- checkpoint first, memo second → crash between → replay redoes X and appends
again → duplicated output.
Duplication is the less bad failure, so ordered writes are a tolerable fallback,
but neither is correct.
**Cost:** `setMemo` now writes a full capped output state rather than a small
value. Memos are rare — a handful per invocation — so this is bounded by
`memo count x cap`, not by output volume.
### 7.6 `openProgress` has a write-ordering race
`commitWrite(item)` captures `item` when `write()` is called, and the write is
fire-and-forget with only `latest` tracked. A checkpoint captured at T1 can
therefore commit *after* a memo-bundled checkpoint at T2, overwriting newer state
with older — reintroducing exactly the loss §7.5 prevents.
Fix: resolve the sink's state inside the command planner rather than at call time.
```ts
commitWrite: () => setValue(address, out.snapshot()) // evaluated under the Session line
```
`lane.command` serializes on the Session line, so checkpoint writes become
monotonic by construction. This removes the race generally, not only against
memos.
### 7.7 Nothing on the durable path runs tool code
Ops are interpreted by a six-verb applier with no domain knowledge, so a tool
rewritten between crash and resume cannot make a persisted stream unreadable.
> **The durable path uses only harness-owned reducers.**
This also rules out persisting *facet* ops. The harness has no facet state,
facets come and go, and a recovering harness must rebuild its working values with
no facet present.
## 8. Open questions
- **Property tests for the tracker** (`delta.md` §3.3). Everything here rests on
producer and replica agreeing; nothing currently proves they do.
- Coalescing window: per tick, or a byte/time threshold.
- Whether image count needs a bound. Images are unwindowed, so a tool pushing
them in a loop grows `content` without limit. Currently treated as a tool bug.
- Whether details need a bound for the same reason (§2.1).
- Whether `retain: "head"` should keep emitting counter-only updates once capped
so a renderer can report how much was suppressed. [`execenv.md`](../03-execenv/execenv.md) answers this
for exec-originated output; agent-side output needs the same answer.
- Whether a failing tool *should* be able to terminate, or whether the current
inability is deliberate — there is a reasonable argument the model should
receive the error and decide.
- **Derived values.** `arguments` parsed from accumulated JSON should not be
replicated; derive it on demand. Safe because `parseStreamingJson` is total —
four fallbacks ending in `{}`, it cannot throw — so a replica derives with no
error path and no agreement protocol. Generalises to: no derived fields in
replicated state.

125
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/rate-limiting.md vendored Normal file
View File

@@ -0,0 +1,125 @@
# Bounded output publication
**Status:** the shared adaptive publisher and its `ExecutionEnv` use are implemented. Generic `ToolOutput` integration remains design work.
Depends on landed Chord delta tracking, source-bounded execution output, and scoped storage for durable batches.
## 1. Invariant
> The durable record, what the model sees, and what the UI shows are the same bounded view.
A spill file is not a second view. It is a file inside the execution environment that the model reaches through ordinary file tools.
Every uncontrolled producer boundary needs two independent bounds:
- **state size:** the latest retained text is capped;
- **publication:** both encoded bytes and event count are paced.
Delta encoding is complementary. It compresses a published change; it does not cap state or decide when publication occurs.
## 2. Boundaries
```text
remote process
-> optional ExecutionEnv publisher
-> worker ToolOutput publisher
-> events + durability + replication
```
A publisher instance is required only before a real costly boundary:
- a physically remote execution environment limits output before transport;
- `ToolOutput` limits custom tools and downstream event/storage traffic;
- a colocated environment can feed its bounded updates in process without another serialized transport;
- custom tools bypass `ExecutionEnv` but cannot bypass `ToolOutput`.
The control algorithm is shared. Payload vocabularies differ: Shell uses replace/append/slide/metadata; `ToolOutput` uses Chord operations.
## 3. Why size or cadence alone is insufficient
A fixed 50 KB snapshot is safe per event but not over time. At 100 ms it permits ten complete snapshots per second, approximately 500 KB/s plus envelopes.
A byte budget alone also permits excessive tiny events and durable transactions. A minimum interval bounds count; encoded-size debt bounds bandwidth.
The original handoff incorrectly treated `intervalMs = 100` as 100 emits/s. It is ten emits/s. The fixed interval was still non-adaptive: it delayed small trickles while allowing complete windows at the same frequency.
## 4. Landed adaptive algorithm
`packages/agent/src/harness/utils/adaptive-publisher.ts` implements:
```ts
nextDelayMs = max(globalMinEmitInterval, encodedUpdateBytes * 1000 / globalTargetBytesPerSecond);
```
Current harness-global policy:
```ts
minEmitInterval = 100 ms;
targetBytesPerSecond = 100 KB/s;
```
Behavior:
1. The first dirty state after idle publishes immediately.
2. Writes before the next deadline collapse into the latest state.
3. One trailing timer publishes held state after the deadline.
4. Completion and correctness boundaries force one bounded publication.
5. The publisher commits its baseline before consumer delivery, preventing duplicate deltas if a consumer applies and then throws.
This is an amortized token bucket with a cap-sized burst. A leading or forced terminal update may exceed the target over a short interval, but sustained encoded bytes converge to the target and sustained event count cannot exceed the minimum-interval floor except for explicit forced correctness writes.
## 5. Scenario traces
Assume a 50 KB cap, 100 KB/s target, and 100 ms floor.
### Below cap, completes
The initial state publishes immediately. Small appends publish no faster than the floor, and final dirty state is forced. Total encoded text is approximately the produced text.
### Below cap, trickling
Writes arriving more than 100 ms apart publish immediately because the preceding deadline has already passed. Faster writes collapse into one append per floor interval.
### Above cap, full force
The retained state never exceeds 50 KB. Complete window turnovers encode as bounded replacements. A roughly 50 KB update buys about 500 ms of silence, yielding approximately two updates and 100 KB per second regardless of raw producer throughput.
For shell execution, the complete stream goes to a source-local spill under backpressure rather than over the output-update channel.
### Above cap, trickling
A small tail movement encodes as truncate plus append (Chord) or `slide` (Shell). Its encoded size is small, so the 100 ms floor dominates and output remains responsive. Resending a complete 50 KB snapshot for each tiny slide would be both slower and larger.
### Burst then silence
The leading state is immediate. Held writes collapse, and one trailing timer publishes the latest residue. There is no polling timer.
### Huge single write
The producer may already have allocated its input, but the boundary retains and publishes only the configured cap. Shell spill keeps the complete source stream. Arbitrary custom-tool images and structured details still require separate limits.
## 6. Forced writes
These bypass pacing once, while remaining state-size bounded:
- command/tool completion;
- error or abort;
- a memo and output checkpoint committed atomically;
- an explicit recovery base/rebase.
A terminal flush cancels the trailing timer before `tool_end`, preventing a late update for a settled invocation.
## 7. `ToolOutput` application
The sink will own one Chord tracker and encoder per invocation. Mutations remain local while publication is blocked; flush-time dirty tracking means held writes collapse without retaining one op per write.
One sink flush feeds the live event and current model-visible state. Durable batches use the same logical flush, encoded as per-stream `WireOp[]`. Periodic `rebase()` bounds recovery replay; a memo correctness flush writes a tagged base batch in the same transaction as the memo.
Durable cadence belongs to the sink, not to Shell or bash. Bash's existing two-second checkpoint request remains only as an interim compatibility mechanism until the sink migration.
## 8. Remaining decisions
- Explicit byte/count rejection policy for images.
- Bounds for structured details; arbitrary JSON cannot be meaningfully tail-windowed.
- Whether ordinary durable writes ride every live sink publication initially or use a slower measured cadence. Memo and terminal correctness flushes are not optional.
- Exact global production values after profiling real remote transport and storage.

227
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/05-assistant-output/message-update.md vendored Normal file
View File

@@ -0,0 +1,227 @@
# `message_update` Write Amplification
> **Scope:** harness-local. Depends on [delta tracking](../01-delta/delta.md) for the landed Chord `Op`/`WireOp` vocabulary and [scoped storage](../02-scopes/scopes.md) for durability. Chord delta tracking has landed; scoped storage and this Harness integration have not.
## 1. The problem
```ts
{ type: "message_update", runId, message, event, frame? }
```
Three representations of the same thing travel together:
- `message` — the full `AssistantMessage`;
- `event` — an `AssistantMessageEvent`, which itself carries `partial: AssistantMessage`,
a second full copy;
- `frame` — the actual delta, optional.
Per streamed token this is roughly two complete snapshots plus a delta, so bytes grow
quadratically over a response.
The reducer does not even use the delta:
```ts
case "message_update":
if (next.operation?.id === event.runId && event.message.role === "assistant") {
next.operation.streamingMessage = event.message;
}
```
A straight assignment. So on any wire, sending the event is *worse* than sending a
snapshot — it ships two where `replace` ships one.
The wire adapter is already halfway to the fix: it drops `event` and sends `message`
plus `frame`. That is the full snapshot together with the delta that produced it.
## 2. The compact form already exists
```ts
/**
* Compact, replayable assistant-message progress. Terminal settlement is
* intentionally excluded and must be persisted separately.
*/
export type AssistantMessageFrame =
| { type: "start"; partial: AssistantMessage }
| { type: "text_start" | "text_delta" | "text_end"; contentIndex: number; ... }
| { type: "thinking_start" | "thinking_delta" | "thinking_end"; ... }
| { type: "toolcall_start" | "toolcall_checkpoint" | "toolcall_delta" | "toolcall_end"; ... };
```
`AssistantMessageFrameEncoder` produces it, `reduceAssistantMessageFrames` folds it,
and `openFrameProgress` already persists it with `appendList`. The delta format is
built, used, and durable. It simply is not what `message_update` carries.
## 3. Precedent
pi already does wire-is-delta one layer down. `PiMessagesEvent` — the serialized form
a pi-messages backend sends — has no `partial`:
```ts
| { type: "text_delta"; contentIndex: number; delta: string }
| { type: "text_end"; contentIndex: number; content: string; contentSignature?: string }
```
`pi-messages.ts` then rehydrates: it holds a local `partial`, mutates it per event
(`partial.content[i].text += event.delta`), and returns the in-process
`AssistantMessageEvent` with `partial` attached.
So the convention is established. It just stops at the provider edge instead of
continuing through the harness.
## 4. The change
```ts
| { type: "message_update"; runId: string; entryId: string; frame: AssistantMessageFrame }
```
`message` and `event` are removed; `frame` stops being optional.
The blast radius is small because `HarnessEvent` is not what most streaming consumers
read. `AgentEvent` (`agent-loop.ts`) and `AgentSessionEvent` (`agent-session.ts`) are
separate unions that happen to share the tag name and build their own
`message_update` from the pi-ai event directly. They are out of scope here.
Real consumers and producers of `HarnessEvent.message_update`:
| site | change |
| --- | --- |
| `runtime/drive/response.ts` | emit the required semantic frame; stop attaching full snapshots |
| `runtime/reducer.ts` | fold the frame instead of assigning `event.message` |
| lane/facet state adapter | run that fold under the Chord tracker and emit `Op[]`/encoded `WireOp[]` |
| `experimental/harness-wire-adapter.ts` | stop treating raw `HarnessEvent` as the final replication format |
| `harness/telemetry.ts` | name list only |
| `protocol/harness.ts` | replication carries encoded `WireOp[]`, not the raw event |
`message_end` continues to carry the settled message, because frames deliberately
exclude terminal settlement. That is once per message, not once per token.
## 5. An incremental applier is needed
`reduceAssistantMessageFrames` is a whole-stream fold over an `Iterable`. The
reducer needs a step function:
```ts
export function applyAssistantMessageFrame(
state: AssistantFrameState,
frame: AssistantMessageFrame,
): void;
```
Plain mutation on a plain object. **No `Draft`, no Immer.** An earlier draft
argued for a draft-mutating signature so it would compose inside a `produce`
recipe; that motivation is gone ([delta.md §8](../01-delta/delta.md#8-what-this-removes-from-the-codebase)). The step function is still
needed — the whole-stream version becomes a loop over it — just for the simpler
reason that the reducer folds one frame at a time.
### 5.1 pi-ai frames stay at the pi-ai boundary; Chord ops cross replication boundaries
`AssistantMessageFrame` is pi-ai's semantic delta vocabulary (`text_delta`, `text_end`, …). [Delta tracking §6](../01-delta/delta.md#6-there-is-no-frame-type) defines no second frame wrapper: in-process replication carries `Op[]`, and a wire adapter carries encoded `WireOp[]`. Pi-ai frames stop at the fold; Chord ops cross the replication boundary.
`AssistantMessageFrame` is pi-ai's own delta vocabulary and stays. What changes is
that it is no longer the durable unit or the replication unit.
The harness folds frames into `LaneView` by plain mutation. Under the Chord tracker that yields:
```json
["a",["operation","streamingMessage","content",0,"text"],"Let me "]
```
Measured, interned ops are **smaller than frames** on this workload — 13.6 KB
against 21.5 KB raw for 200 deltas — because a frame carries three keys of its own
where an interned op carries one integer. So the size argument for putting frames
on the wire is dead.
What frames keep is semantic: `text_end` carries authoritative content plus a
signature in one atomic unit, where ops would need two with a weaker contract
about their relationship. That is why they remain the *input* vocabulary and are
folded before anything crosses a boundary.
### 5.2 Reducer state must live in the reduced value
Text and thinking fold purely: `block.text += frame.delta`, and `*_end` overwrites
with authoritative content plus signature. The `ended` flag in `ReducerBlockState`
is used only for validation and can be dropped.
Tool calls cannot. `toolcall_delta` does `state.json += frame.delta`, accumulating
a **raw JSON string that is never stored on the message** — the block holds
`arguments`, the parsed value. You cannot recover the accumulator from the
snapshot, and you cannot append a delta to a parsed object.
So the accumulator has to become part of the value being folded — e.g.
`operation.frameState[contentIndex].json` on `LaneView` — leaving
`AssistantMessage` clean. Generally:
> **A replicated reducer's state must be part of the replicated value.** Anything
> held beside it diverges on any consumer that did not run the producer's fold.
The corollary is that `arguments` itself should **not** be replicated. It is
derived from `json`, and a parsed object is a fresh reference on every parse, so
storing both ships two copies of the same information. Derive it on demand.
This is safe precisely because `parseStreamingJson` is **total** — four fallbacks
ending in `{}`, it cannot throw — so a replica derives unconditionally with no
error path and no agreement protocol. There is no parse-failure state to
represent, and no block needs an error slot.
### 5.3 Parse cost
`parseStreamingJson` on a growing string once per delta is quadratic per message.
Since `arguments` is now derived rather than replicated, this cost falls on whoever reads it rather than on every consumer. Presentation can refresh derived arguments at semantic checkpoints and `toolcall_end` instead of parsing on every delta; that policy is separate from why the encoder currently emits a checkpoint (§6).
## 6. What `toolcall_checkpoint` is for
`EncoderBlockState` carries `caughtUp` and `catchupJson` because a queued provider event's shared `partial` may already be ahead of that event's delta. The checkpoint catches the semantic frame stream up to the authoritative tool-call arguments visible at block start; it is not currently a general late-subscriber protocol.
After frames fold into tracked state, a Chord root replacement (`r`) is the replication and durable-recovery resync point. `toolcall_checkpoint` remains semantic input to that fold rather than carrying transport responsibility.
## 7. Write volume for pending output
`openFrameProgress` calls `appendList(pendingAssistantFrames(...))`, so every
frame is a line. A long response is thousands of writes.
**The address is renamed `pendingAssistantOutput`** and becomes a `list<WireOp[]>`, matching `pendingToolOutput` ([tool-output handoff §7.2](../04-tool-output/harness-tools.md#72-renaming)), and it stops being a list of frames. The progress sink encodes tracked `Op[]` with one stateful encoder per response before appending each durable `WireOp[]` batch; explicit `rebase()` calls produce bounded root-replacement batches for recovery. The list lives in an **ephemeral scope** so it is unlinked on settle rather than persisting in the main log ([scoped storage](../02-scopes/scopes.md)).
The important property is that this list is **not history**: `deleteList` runs on
settle in `response.ts`, `deferred.ts`, and `terminal.ts`. It exists so a crash
mid-response can recover a partial assistant message. The settled message is persisted
separately.
That means per-frame durability buys almost nothing, and the write rate can be traded
away directly:
- **Coalesce with a bounded, non-resetting window.** The first pending frame opens the
window; later frames join it *without* extending the deadline, so a continuously
streaming response cannot postpone the first write indefinitely — the failure mode a
naive debounce has. Frames admitted during an active write form the next batch.
- **Concatenate on flush.** A run of `text_delta` frames for one `contentIndex`
collapses to a single frame with the joined text. The fold result is identical, so
this is lossless for our purposes.
A crash then loses at most one window of in-flight streaming.
### 7.1 Contrast with DeepSeek Harness
DSH cannot make this trade. Its `assistant/chunk` events are canonical log entries, so
per-token durability is mandatory and it attacks size instead:
- the same bounded non-resetting write-behind window;
- **packed rows** — runs of consecutive chunk deltas stored as
`text-chunks` / `reasoning-chunks` / `tool-call-chunks`, lossless and roughly 60%
smaller on a real session, with reading unconditional so layout never depends on the
write switch;
- checksummed zstd frames by default, with recovery from a torn final frame.
Their packing must reconstruct exact event boundaries, sequence numbers and timestamps,
because `seq = log.length` and validation requires a contiguous logical log. Ours does
not, because the frames are discarded at settle — so plain concatenation is available
to us and packing machinery is not needed.
## 8. Consequence for ad-hoc listeners
`HarnessEvent.message_update` stops being self-describing. A listener that attaches
mid-stream sees a delta against a partial it does not hold.
Any correct listener already has the base, because `watch()` installs the subscription
and captures the snapshot inside one `readLane` critical section, buffering until
`start()`. But a listener that calls `on("message_update")` directly, without a watch,
is no longer viable — worth knowing before committing.

1855
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/01-facets/facets.md vendored Normal file
View File

File diff suppressed because it is too large Load Diff

162
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/README.md vendored Normal file
View File

@@ -0,0 +1,162 @@
# Facet sandbox — isolated-vm
**Unmodified pi facet code running in a V8 isolate with no ambient authority.**
Full JIT, so ~1.8× native rather than the 8–17× a WASM interpreter costs.
```bash
npm install
npm run demo # a facet: contributions, components, commands, callbacks
npm run audit # escape audit — the interesting one
npm run bench # crossing cost, isolate cost, 300 components
npm test # 412 property assertions
```
Node 22+ (`--experimental-strip-types`). See `../facet-sandbox-poc` for the
QuickJS-WASM equivalent.
## The safety claim, and why it is structural
isolated-vm **can** be used unsafely, and that is what `facets.md` §14.2 objected
to. The footgun is handing the guest a live handle to a host object — via
`derefInto()`, or by returning a `Reference` from a host call — after which the
guest walks that object's prototype chain into the host realm.
This membrane makes that impossible **by construction, not by care**:
1. `encode()` is the only way a host value reaches the guest, and it is
`JSON.stringify` with a replacer. **Its output is a string.** A string cannot
carry a reference.
2. Host callables never cross. They become an integer id into `hostTable`, which
lives entirely host-side. The guest receives a **number**.
3. Exactly **one** `Reference` is ever given to the guest — `__invokeRef`, the
single call-in point.
4. `derefInto()` is used exactly once, on the guest's **own** global. No host
object is ever its argument.
The guest's entire view of the host is `{ number, string }`. There is no object
graph, so there is nothing to walk.
### Audit results (`npm run audit`)
```
Ambient authority:
require / process / fetch ["undefined","undefined","undefined"]
global names visible 64 globals
Classic escape ladders:
Function('return process')() undefined
constructor walk on host data undefined
The isolated-vm Reference footgun:
__invokeRef.deref() blocked: TypeError ("Cannot dereference
this from current isolate")
__invokeRef.copySync() blocked: TypeError
__invokeRef.getSync('constructor') blocked: TypeError
derefInto() is callable? inert (object)
invoke derefInto() result blocked: TypeError
host globals via derefInto() no host globals
Resource bounds:
spinning guest interrupted after 205ms
isolate usable afterwards true
```
One probe gave a **false positive** in a first draft: testing `f.deref === undefined`
to prove a returned host function is a Proxy rather than a Reference. The
membrane's proxy returns a proxy for *every* property, so `f.deref` is truthy.
That is not a leak — calling it routes to `hostFn["deref"]`, which does not exist
and throws host-side. The audit now calls it instead of checking for absence.
## Performance
| | isolated-vm | QuickJS | native |
| --- | --- | --- | --- |
| 300 markdown components, small | **70 ms** | 494 ms | ~40 ms |
| 300 markdown components, typical | **108 ms** | 1118 ms | 64 ms |
| 300 markdown components, large | **419 ms** | 4260 ms | ~150 ms |
| 146 KB bundle load | **35 ms** | 104 ms | — |
| membrane crossing | **4.5 µs** | 7–17 µs | — |
| per compartment | 1080 KB, 5.5 ms | **77 KB, 0.8 ms** | — |
**~1.8× native at typical size.** QuickJS is 8–17× and gets worse with input size.
Isolates cost **14× more memory than a QuickJS runtime** (1080 KB vs 77 KB) and
are slower to create. For a handful of facets that is irrelevant; for hundreds it
would not be.
`Intl.Segmenter` **exists** here, which is the single hard blocker for reusing
`packages/tui` components under QuickJS. That alone may decide it.
## Limitations
**Budgets do not nest.** A timeout bounds one `evalSync`. A guest function
re-entered *by the host* — a contributed callback, a component method — gets
`callGuestRef`'s own budget, not the outer one. Asserted explicitly in
`property-test.ts`; a runaway callback took 5005 ms under an outer budget of 50 ms.
Bounding total facet time needs separate accounting.
**Async is a settle-callback, not a native promise.** `applySync` is synchronous,
so a host Promise cannot cross. The host returns a token, the guest builds a real
Promise against it, and the host settles it by calling back in. Verified: two
sequential guest `await`s while the host event loop ticked 22 times.
**Native addon, and the ABI matrix is real.** `isolated-vm` ships prebuilds and
only falls back to `node-gyp rebuild` when none matches — which is why install
takes a second, not an hour. It is **not** building V8; V8 is already in the Node
binary. But coverage is narrow:
| version | engines | prebuilds |
| --- | --- | --- |
| **6.2.0** | `>=22.0.0` | linux-x64/arm64, darwin-arm64, win32-x64 — abi127, abi137 |
| 7.0.1 | `>=24.0.0` | — |
| 7.0.0 | `>=26.0.0` | — |
darwin-**x64** is absent everywhere. Installing `isolated-vm@7` on Node 22 falls
through to a source build and **fails without Python and a C++ toolchain** —
verified here. Note also that 6.2.0 is `latest` on npm despite 7.0.1 existing,
because it published one day later.
The project is **not abandonware** — 6.0.1 (Jul 2025) through 7.0.1 (Aug 2026),
actively released. §14.2's "maintenance mode" claim is out of date and should be
corrected. The real risk is different: adopting it couples your minimum Node
version to theirs, and they dropped Node 20 in 6.x and Node 22 in 7.x within a
year. Shipping SEA builds moves that from every user's install to your CI.
**Memory: references release, but lazily.** The `WeakRef` + `FinalizationRegistry`
interning table ports unchanged from the QuickJS membrane and works — 20 000
crossings went to **0 live refs**. But finalization is not prompt: 5 000 refs were
still live 200 ms after a forced GC. Under sustained churn refs accumulate faster
than the collector reclaims them, and a 32 MB isolate did hit its limit.
The consequence is an API rule, not a membrane fix: **do not create references in
a hot path.** `slots.claim(factory)` at construct is one reference forever; a
component method returning a fresh closure every frame is one per frame.
**Hitting `memoryLimit` kills the facet cleanly.** It throws a catchable
`"Isolate was disposed during execution due to memory limit"` and the host
survives — but the isolate is dead and unrecoverable, so teardown must tolerate
an already-disposed isolate. `dispose()` on it throws `"Isolate is already
disposed"`; the membrane guards this.
> **Do not call `isolate.getHeapStatisticsSync()` on a stressed isolate.** It
> **aborted the whole process** during testing — a hard crash, not an exception.
> A `heapMB()` helper was removed from the membrane rather than shipped.
**Still one process, one engine.** A separate isolate is a far stronger boundary
than SES, but Figma's argument for QuickJS was that a *different VM* cannot
confuse objects because the representations differ. Here the guarantee is V8's
isolate boundary plus the membrane's discipline — strong, and structurally
enforced above, but not the same category of claim.
## Files
- `src/membrane.ts` — the membrane. The safety argument is in the header comment.
- `src/facet-example.js` — unmodified facet code: `ctx.use()`, `slots.claim()`
with a closure, a class instance returned to the host, a subscribe callback.
- `src/demo-facet.ts` — loads it and exercises four boundary crossings.
- `src/escape-audit.ts` — the audit above.
- `src/property-test.ts` — 412 assertions: random `JsonValue` round-trips both
directions, identity across crossings, callables, error propagation, reference
release under GC, prototype pollution both ways, budgets, async, disposal.
- `src/bench.ts` — the numbers above.
- `src/markdown-bundle.js` — the real pi `Markdown` component, esbuild-bundled.

13
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/package.json vendored Normal file
View File

@@ -0,0 +1,13 @@
{
"name": "facet-sandbox-ivm",
"private": true,
"type": "module",
"description": "Running pi facets in an isolated-vm isolate: full JIT, no ambient authority.",
"scripts": {
"demo": "node --experimental-strip-types src/demo-facet.ts",
"audit": "node --experimental-strip-types src/escape-audit.ts",
"bench": "node --experimental-strip-types src/bench.ts",
"test": "node --experimental-strip-types --expose-gc src/property-test.ts"
},
"dependencies": { "isolated-vm": "6.2.0" }
}

48
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/bench.ts vendored Normal file
View File

@@ -0,0 +1,48 @@
/**
* Cost of the isolate, the membrane, and a real component at transcript scale.
* Compare against `../facet-sandbox-poc` (QuickJS) and native.
*/
import ivm from "isolated-vm";
import { readFileSync } from "node:fs";
import { createCompartment } from "./membrane.ts";
// ── membrane crossing ─────────────────────────────────────────────────────
{
const c = await createCompartment();
await c.endow("__h", { echo: (v: unknown) => v });
c.load(`globalThis.spin = function (n) { for (var i = 0; i < n; i++) __h.echo("x"); };`);
c.load(`spin(2000);`);
const N = 20000, t = Date.now();
c.load(`spin(${N});`, 60000);
console.log(`membrane crossing: ${((Date.now() - t) * 1000 / N).toFixed(1)} µs`);
c.dispose();
}
// ── isolate creation ──────────────────────────────────────────────────────
{
const before = process.memoryUsage().rss, t = Date.now();
const held = Array.from({ length: 50 }, () => {
const iso = new ivm.Isolate({ memoryLimit: 32 });
const ctx = iso.createContextSync();
ctx.evalSync(`var c = { render: function () { return ["x"]; } };`);
return iso;
});
console.log(`50 isolates: ${(((process.memoryUsage().rss - before) / 50) / 1024).toFixed(0)} KB each, ${Date.now() - t} ms total`);
for (const i of held) i.dispose();
}
// ── real Markdown at transcript scale ─────────────────────────────────────
{
const iso = new ivm.Isolate({ memoryLimit: 512 });
const ctx = iso.createContextSync();
ctx.global.setSync("global", ctx.global.derefInto());
const t = Date.now();
ctx.evalSync(readFileSync(new URL("./markdown-bundle.js", import.meta.url), "utf8"), { timeout: 60000 });
console.log(`146 KB bundle load: ${Date.now() - t} ms`);
for (const [label, pts] of [["small", 2], ["typical", 8], ["large", 35]] as [string, number][]) {
ctx.evalSync(`__bench(${pts}, 20, 50)`, { timeout: 120000 }); // warm the JIT
const ms = ctx.evalSync(`__bench(${pts}, 300, 3)`, { timeout: 300000 });
console.log(`300 markdown components, ${label.padEnd(8)} ${ms.toFixed(0)} ms per full repaint`);
}
iso.dispose();
}

22
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/demo-facet.ts vendored Normal file
View File

@@ -0,0 +1,22 @@
import { readFileSync } from "node:fs";
import { createCompartment } from "./membrane.ts";
const mounted = new Map(), commands = new Map(), notifications = [], subs = [];
const Tui = {
slots: { claim: (s, f) => mounted.set(s, f) },
commands: { add: (n, h) => commands.set(n, h) },
notify: (m) => notifications.push(m),
};
const Transcript = { tail: { subscribe: (f) => subs.push(f) } };
const c = await createCompartment();
await c.endow("__host", { use: (t) => (t === "Tui" ? Tui : Transcript) });
c.load(readFileSync(new URL("./facet-example.js", import.meta.url), "utf8"));
c.load(`globalThis.__r = __facet.construct({ use: function (t) { return __host.use(t); } });`);
console.log("contributions:", { slots: [...mounted.keys()], commands: [...commands.keys()], subs: subs.length });
const comp = mounted.get("footer")({ label: "presses" });
console.log(comp.render(30).join("\n"));
comp.handleInput("abc");
console.log(comp.render(30).join("\n"));
console.log("command ->", commands.get("footer.reset")());
subs[0]({ entries: [1, 2, 3] });
console.log("notifications:", notifications);
c.dispose();

63
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/escape-audit.ts vendored Normal file
View File

@@ -0,0 +1,63 @@
/**
* Escape audit.
*
* The claim the membrane makes is that the guest's entire view of the host is
* `{ number, string }` — no object graph, so nothing to walk. This tries to
* break that, including the specific `Reference` footgun that makes isolated-vm
* dangerous when used naively.
*/
import { createCompartment } from "./membrane.ts";
const c = await createCompartment();
const seen: string[] = [];
await c.endow("__h", {
sink: (v: string) => seen.push(v),
give: () => ({ nested: { a: 1 } }),
giveFn: () => () => "host function result",
});
const probe = (name: string, src: string) => {
c.load(`globalThis.__p = String(${src});`);
c.load(`__h.sink(__p);`);
const got = seen.pop();
const bad = got !== undefined && /GOT (?!undefined)|INVOKED|CALLABLE|^object$/.test(got) && !/blocked/.test(got);
console.log(` ${bad ? "!!" : "ok"} ${name.padEnd(34)} ${got}`);
return !bad;
};
console.log("Ambient authority:");
let ok = true;
ok = probe("require / process / fetch", `JSON.stringify([typeof require, typeof process, typeof fetch])`) && ok;
ok = probe("global names visible", `Object.getOwnPropertyNames(globalThis).length + " globals"`) && ok;
console.log("\nClassic escape ladders:");
ok = probe("Function('return process')()", `(function(){ try { return String((function(){}).constructor("return typeof process")()); } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("constructor walk on host data", `(function(){ try { var o = __h.give(); return String(o.constructor.constructor("return typeof process")()); } catch(e){ return "blocked: "+e.name; } })()`) && ok;
console.log("\nThe isolated-vm Reference footgun:");
ok = probe("__invokeRef.deref()", `(function(){ try { return "GOT " + typeof __invokeRef.deref(); } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("__invokeRef.copySync()", `(function(){ try { return "GOT " + typeof __invokeRef.copySync(); } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("__invokeRef.getSync('constructor')", `(function(){ try { return "GOT " + typeof __invokeRef.getSync("constructor"); } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("derefInto() is callable?", `(function(){ var d = __invokeRef.derefInto(); return typeof d === "function" ? "CALLABLE" : "inert (" + typeof d + ")"; })()`) && ok;
ok = probe("invoke derefInto() result", `(function(){ try { __invokeRef.derefInto()(); return "INVOKED"; } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("host globals via derefInto()", `(function(){ var d = __invokeRef.derefInto(); return String(d.process || d.global || "no host globals"); })()`) && ok;
console.log("\nWhat a host FUNCTION looks like to the guest:");
ok = probe("typeof a returned host fn", `(function(){ var f = __h.giveFn(); return typeof f; })()`) && ok;
// NOTE: `f.deref` is NOT undefined — the membrane's proxy returns a proxy for
// every property. That is not a leak: calling it routes to hostFn["deref"],
// which does not exist, and throws host-side. Testing for the property's
// absence gave a false positive in a first draft of this audit.
ok = probe("calling f.deref() reaches nothing", `(function(){ try { var f = __h.giveFn(); f.deref(); return "INVOKED"; } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("calling the host fn works", `(function(){ var f = __h.giveFn(); return String(f()); })()`) && ok;
console.log("\nResource bounds:");
const t0 = Date.now();
let interrupted = false;
try { c.load(`while(true){}`, 200); } catch { interrupted = true; }
console.log(` ${interrupted ? "ok" : "!!"} spinning guest interrupted after ${Date.now() - t0}ms`);
console.log(` ok isolate usable afterwards ${c.load(`1+1`) === 2}`);
c.dispose();
console.log(`\n${ok && interrupted ? "AUDIT PASSED" : "AUDIT FAILED"}`);
process.exit(ok && interrupted ? 0 : 1);

42
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/facet-example.js vendored Normal file
View File

@@ -0,0 +1,42 @@
// ── REAL facet code. Nothing here knows it is sandboxed. ────────────────────
// No imports rewritten, no string marshalling, no __dispatch. It uses the
// host API exactly as facets.md §11.1 describes it.
class FooterComponent {
constructor(props) { this.label = props.label; this.count = 0; }
render(width) {
const text = ` ${this.label}: ${this.count} `;
const pad = Math.max(0, width - text.length - 2);
const bar = "\u2500".repeat(Math.max(0, width - 2));
return [`\u250c${bar}\u2510`, `\u2502${text}${" ".repeat(pad)}\u2502`, `\u2514${bar}\u2518`];
}
handleInput(data) { this.count += data.length; }
}
const facet = {
id: "@demo/footer:tui",
uses: ["Tui", "Transcript"],
construct(ctx) {
const tui = ctx.use("Tui");
const transcript = ctx.use("Transcript");
// A factory crossing guest → host, by reference.
tui.slots.claim("footer", (props) => new FooterComponent(props));
// A command handler, likewise.
tui.commands.add("footer.reset", () => {
tui.notify("footer reset");
return "ok";
});
// A subscription: host calls back into the guest.
transcript.tail.subscribe((tail) => {
tui.notify(`transcript now has ${tail.entries.length} entries`);
});
return [];
},
};
globalThis.__facet = facet;

34
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/iso-vs-ctx.ts vendored Normal file
View File

@@ -0,0 +1,34 @@
import ivm from "isolated-vm";
const rss = () => process.memoryUsage().rss / 1024 / 1024;
// 1. Are two CONTEXTS in one isolate isolated from each other?
{
const iso = new ivm.Isolate({ memoryLimit: 64 });
const a = iso.createContextSync(), b = iso.createContextSync();
a.evalSync(`globalThis.secret = "from-A"`);
console.log("context B sees A's global:", b.evalSync(`typeof globalThis.secret`));
// Can a spin in A be timed out without harming B?
try { a.evalSync(`while(true){}`, { timeout: 100 }); } catch { /* expected */ }
console.log("B usable after A was interrupted:", b.evalSync(`1+1`) === 2);
// Does A's memory bomb kill B?
try { a.evalSync(`var x=[]; for(;;) x.push(new Array(50000).fill(1));`, { timeout: 15000 }); }
catch (e: any) { console.log("A bomb:", e.message.slice(0, 45)); }
try { console.log("B after A's bomb:", b.evalSync(`1+1`)); }
catch (e: any) { console.log("B after A's bomb: DEAD -", e.message.slice(0, 40)); }
try { iso.dispose(); } catch {}
}
// 2. Cost per facet, both ways.
for (const mode of ["one isolate, N contexts", "N isolates"]) {
const before = rss(), t = Date.now();
const held: any[] = [];
const shared = mode.startsWith("one") ? new ivm.Isolate({ memoryLimit: 512 }) : null;
for (let i = 0; i < 50; i++) {
const iso = shared ?? new ivm.Isolate({ memoryLimit: 32 });
const ctx = iso.createContextSync();
ctx.evalSync(`var c = { render: function () { return ["x"]; } };`);
held.push(iso);
}
console.log(`50 facets, ${mode.padEnd(24)}: ${(((rss() - before) / 50) * 1024).toFixed(0)} KB each, ${Date.now() - t} ms`);
if (shared) { try { shared.dispose(); } catch {} } else for (const i of held) { try { i.dispose(); } catch {} }
}

27
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/leak.ts vendored Normal file
View File

@@ -0,0 +1,27 @@
import { createCompartment } from "./membrane.ts";
const c = await createCompartment();
let seen = 0;
await c.endow("__l", { drop: (_fn: unknown) => { seen++; } });
c.load(`globalThis.churn = function (n) { for (var i = 0; i < n; i++) __l.drop(function () { return i; }); };`);
const rss = () => process.memoryUsage().rss / 1024 / 1024;
const before = rss();
c.load(`churn(20000);`, 60000);
const peak = c.debugRefCount();
const afterChurn = rss();
globalThis.gc?.(); await new Promise((r) => setTimeout(r, 100));
globalThis.gc?.(); await new Promise((r) => setTimeout(r, 100));
const settled = c.debugRefCount();
console.log(`20000 crossings: host called ${seen}x`);
console.log(` guest refs: ${peak.guestRefs} -> ${settled.guestRefs}`);
console.log(` host table: ${peak.hostTable} -> ${settled.hostTable}`);
console.log(` rss: ${before.toFixed(0)} -> ${afterChurn.toFixed(0)} -> ${rss().toFixed(0)} MB`);
// Cost of the release path: one evalSync crossing per finalized ref.
const t = Date.now();
c.load(`churn(5000);`, 60000);
globalThis.gc?.(); await new Promise((r) => setTimeout(r, 200));
console.log(` 5000 more + gc: ${Date.now() - t} ms, refs now ${c.debugRefCount().guestRefs}`);
c.dispose();

3888
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/markdown-bundle.js vendored Normal file
View File

File diff suppressed because it is too large Load Diff

197
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/membrane.ts vendored Normal file
View File

@@ -0,0 +1,197 @@
import ivm from "isolated-vm";
/**
* A membrane between the host realm and an isolated-vm isolate.
*
* ── The safety argument, which is structural, not a convention ──────────────
*
* isolated-vm CAN be used unsafely. The documented footgun is handing the guest
* a live handle to a host object — via `derefInto()`, or by returning a
* `Reference` from a host call — after which the guest can walk that object's
* prototype chain into the host realm.
*
* This membrane makes that impossible by construction rather than by care:
*
* 1. `encode()` is the ONLY way a host value reaches the guest, and it is
* `JSON.stringify` with a replacer. Its output is a string. A string
* cannot carry a reference.
* 2. Host callables do not cross. They are replaced by an integer id into
* `hostTable`, which lives entirely host-side. The guest receives a number.
* 3. Exactly ONE `Reference` is ever given to the guest — `__invokeRef`, the
* single call-in point — and its `deref()` throws cross-isolate ("Cannot
* dereference this from current isolate"). Its `derefInto()` yields an
* inert `Dereference` marker: not callable, no host globals.
* 4. `derefInto()` is used exactly once, on the guest's OWN global. No host
* object is ever an argument to it.
*
* The consequence: the guest's entire view of the host is `{ number, string }`.
* There is no object graph to walk, so there is nothing to walk it into. This is
* the same property the QuickJS membrane has, and it is why the choice between
* them is about performance and packaging rather than about safety.
*
* Verified by `src/escape-audit.ts`.
*/
export async function createCompartment({ memoryLimit = 256 } = {}) {
const isolate = new ivm.Isolate({ memoryLimit });
const ctx = await isolate.createContext();
await ctx.global.set("global", ctx.global.derefInto()); // guest's own global only
const hostTable = new Map();
let nextHostId = 1;
const exposeHost = (o) => { const id = nextHostId++; hostTable.set(id, o); return id; };
const guestProxies = new Map();
const finalizer = new FinalizationRegistry((id) => {
guestProxies.delete(id);
if (!released) ctx.evalSync(`delete __refs[${JSON.stringify(String(id))}]`);
});
let released = false;
// Async: `applySync` is synchronous, so a host Promise cannot itself cross.
// Instead the host returns a token, the guest builds a real Promise against
// it, and the host settles that Promise by calling back in when it resolves.
let nextPending = 1;
function encodePending(promise) {
const token = nextPending++;
promise.then(
(v) => settle(token, true, v === undefined ? "" : encode(v)),
(e) => settle(token, false, JSON.stringify(String(e?.message ?? e))),
);
return JSON.stringify({ __pending: token });
}
function settle(token, ok, json) {
if (released) return;
ctx.evalSync(`__settle(${token}, ${ok}, ${JSON.stringify(json)})`);
}
function encode(v) {
if (v && typeof v.then === "function") return encodePending(v);
return JSON.stringify(v, (_k, val) => {
if (typeof val === "function") return { __ref: exposeHost(val) };
if (val && typeof val === "object" && !Array.isArray(val) && hasMethods(val))
return { __obj: exposeHost(val), keys: methodNames(val) };
return val;
});
}
const decode = (json) => JSON.parse(json, (_k, v) => reviveGuest(v));
function intern(id, build) {
const hit = guestProxies.get(id)?.deref();
if (hit !== undefined) return hit;
const p = build();
guestProxies.set(id, new WeakRef(p));
finalizer.register(p, id);
return p;
}
function reviveGuest(v) {
if (!v || typeof v !== "object") return v;
if (typeof v.__guestRef === "number") { const r = v.__guestRef; return intern(r, () => (...a) => callGuestRef(r, a)); }
if (typeof v.__guestObj === "number") {
const r = v.__guestObj;
return intern(r, () => { const o = {}; for (const k of v.keys) o[k] = (...a) => callGuestRef(r, [k, a]); return o; });
}
return v;
}
function callGuestRef(ref, args, timeout = 5000) {
const out = ctx.evalSync(
`__refs[${JSON.stringify(String(ref))}](${JSON.stringify(JSON.stringify(args))})`,
{ timeout },
);
return out === "" || out === undefined ? undefined : decode(out);
}
// The single host entry point. A Reference, applied synchronously with copied
// arguments — no host object graph is ever exposed.
const invoke = new ivm.Reference((idJson, pathJson, argsJson) => {
const path = JSON.parse(pathJson);
let target = hostTable.get(JSON.parse(idJson));
for (let i = 0; i < path.length - 1; i++) target = target[path[i]];
const result = path.length ? target[path[path.length - 1]](...decode(argsJson)) : target(...decode(argsJson));
return result === undefined ? "" : encode(result);
});
await ctx.global.set("__invokeRef", invoke);
await ctx.eval(`
globalThis.__refs = {}; globalThis.__nextRef = 1;
globalThis.__outgoing = new WeakMap();
globalThis.__methodNames = function (o) {
var out = [], p = o;
while (p && p !== Object.prototype) {
for (const k of Object.getOwnPropertyNames(p))
if (k !== "constructor" && typeof o[k] === "function" && out.indexOf(k) < 0) out.push(k);
p = Object.getPrototypeOf(p);
}
return out;
};
globalThis.__mkRef = function (fn) {
var prior = __outgoing.get(fn); if (prior !== undefined) return { __guestRef: prior };
var id = __nextRef++; __outgoing.set(fn, id);
__refs[String(id)] = function (j) { var r = fn.apply(null, JSON.parse(j)); return r === undefined ? "" : __encode(r); };
return { __guestRef: id };
};
globalThis.__mkObjRef = function (obj) {
var prior = __outgoing.get(obj);
if (prior !== undefined) return { __guestObj: prior, keys: __methodNames(obj) };
var id = __nextRef++; __outgoing.set(obj, id);
__refs[String(id)] = function (j) { var c = JSON.parse(j); var r = obj[c[0]].apply(obj, c[1]); return r === undefined ? "" : __encode(r); };
return { __guestObj: id, keys: __methodNames(obj) };
};
globalThis.__encode = function (v) {
return JSON.stringify(v, function (k, val) {
if (typeof val === "function") return __mkRef(val);
if (val && typeof val === "object" && !Array.isArray(val) && __methodNames(val).length) return __mkObjRef(val);
return val;
});
};
globalThis.__pending = {};
globalThis.__settle = function (token, ok, json) {
var d = __pending[token];
if (!d) return;
delete __pending[token];
var value = json === "" ? undefined : JSON.parse(json);
ok ? d.resolve(__revive(value)) : d.reject(new Error(value));
};
globalThis.__revive = function (v) {
if (v && typeof v === "object" && typeof v.__pending === "number") {
var token = v.__pending;
return new Promise(function (res, rej) { __pending[token] = { resolve: res, reject: rej }; });
}
if (v && typeof v === "object") {
if (typeof v.__ref === "number") return __wrap(v.__ref, []);
if (typeof v.__obj === "number") return __wrap(v.__obj, []);
for (var k in v) v[k] = __revive(v[k]);
}
return v;
};
globalThis.__wrap = function (id, path) {
return new Proxy(function () {}, {
get: function (t, prop) {
if (typeof prop !== "string") return undefined;
if (prop === "then" || prop === "toJSON" || prop === "inspect") return undefined;
if (prop === "length" || prop === "name") return t[prop];
return __wrap(id, path.concat([prop]));
},
apply: function (_t, _this, args) {
var out = __invokeRef.applySync(undefined, [JSON.stringify(id), JSON.stringify(path), __encode(args)]);
return out === "" ? undefined : __revive(JSON.parse(out));
},
});
};
`);
return {
async endow(name, obj) { await ctx.eval(`globalThis[${JSON.stringify(name)}] = __wrap(${exposeHost(obj)}, []);`); },
load: (src, timeout = 30000) => ctx.evalSync(src, { timeout }),
debugRefCount: () => ({ hostTable: hostTable.size, guestRefs: ctx.evalSync(`Object.keys(__refs).length`) }),
dispose() {
released = true;
// An isolate that hit its memory limit is already disposed, and calling
// dispose() again throws "Isolate is already disposed".
try { isolate.dispose(); } catch { /* already gone */ }
},
};
}
const RESERVED = new Set(["constructor"]);
const methodNames = (o) => { const out = []; for (let p = o; p && p !== Object.prototype; p = Object.getPrototypeOf(p)) for (const k of Object.getOwnPropertyNames(p)) if (!RESERVED.has(k) && typeof o[k] === "function" && !out.includes(k)) out.push(k); return out; };
const hasMethods = (o) => methodNames(o).length > 0;

137
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/property-test.ts vendored Normal file
View File

@@ -0,0 +1,137 @@
/**
* Property tests for the membrane.
*
* Every bug found by hand while building it failed SILENTLY — wrong output, no
* throw. That is the worst possible failure mode for a boundary, and the reason
* these exist. Run with: node --experimental-strip-types --expose-gc property-test.ts
*/
import { createCompartment } from "./membrane.ts";
let pass = 0;
const failures: string[] = [];
const check = (name: string, ok: boolean, detail = "") => {
if (ok) pass++;
else failures.push(`${name}${detail ? ` — ${detail}` : ""}`);
};
/** Random JsonValue. The membrane claims to round-trip exactly these. */
function randomJson(depth = 0): unknown {
const r = Math.random();
if (depth > 3 || r < 0.25) return Math.floor(Math.random() * 1000) - 500;
if (r < 0.35) return Math.random() * 1e6 - 5e5;
if (r < 0.45) return Math.random() < 0.5;
if (r < 0.5) return null;
if (r < 0.65) {
const alphabet = ['a', 'z', '0', ' ', '"', '\\', '\n', '\u00e9', '\u4e2d', '\u{1f600}'];
let s = "";
const n = Math.floor(Math.random() * 12);
for (let i = 0; i < n; i++) s += alphabet[Math.floor(Math.random() * alphabet.length)];
return s;
}
if (r < 0.8) return Array.from({ length: Math.floor(Math.random() * 5) }, () => randomJson(depth + 1));
const o: Record<string, unknown> = {};
for (const k of ["a", "b", "c", "d", "__proto__x", "0", "length"]) {
if (Math.random() < 0.5) o[k] = randomJson(depth + 1);
}
return o;
}
const c = await createCompartment();
// ── 1. Value round-trip, both directions ──────────────────────────────────
const echoed: unknown[] = [];
await c.endow("__h", { echo: (v: unknown) => v, sink: (v: unknown) => { echoed.push(v); } });
c.load(`globalThis.roundTrip = function (v) { return __h.echo(v); };
globalThis.sendBack = function (v) { __h.sink(v); };`);
for (let i = 0; i < 400; i++) {
const value = randomJson();
const json = JSON.stringify(value);
if (json === undefined) continue;
// host -> guest -> host
c.load(`globalThis.__t = roundTrip(${json});`);
c.load(`sendBack(__t);`);
const got = echoed.pop();
check("value round-trip", JSON.stringify(got) === json, `${json} became ${JSON.stringify(got)}`);
}
// ── 2. Identity: the same guest object must map to the same host proxy ────
const seen: unknown[] = [];
await c.endow("__id", { take: (o: unknown) => { seen.push(o); } });
c.load(`globalThis.shared = { m: function () { return 1; } };
globalThis.sendShared = function () { __id.take(shared); __id.take(shared); };`);
c.load(`sendShared();`);
check("identity preserved across crossings", seen[0] === seen[1]);
c.load(`sendShared();`);
check("identity stable across separate calls", seen[0] === seen[2]);
// ── 3. Callables survive and remain callable ──────────────────────────────
let held: any;
await c.endow("__f", { keep: (fn: unknown) => { held = fn; } });
c.load(`__f.keep(function (a, b) { return a + b; });`);
check("guest function callable from host", held(2, 3) === 5, `got ${held?.(2, 3)}`);
// ── 4. Errors cross with name and message ─────────────────────────────────
c.load(`__f.keep(function () { throw new TypeError("boom"); });`);
let msg = "";
try { held(); } catch (e: any) { msg = e.message; }
check("guest throw reaches host", msg.includes("boom"), `got "${msg}"`);
// ── 5. Reference release: crossings must not leak ─────────────────────────
await c.endow("__l", { drop: (_fn: unknown) => {} });
c.load(`globalThis.churn = function () { for (var i = 0; i < 500; i++) __l.drop(function () { return i; }); };`);
c.load(`churn();`);
const beforeGc = c.debugRefCount().guestRefs;
globalThis.gc?.();
await new Promise((r) => setTimeout(r, 60));
globalThis.gc?.();
await new Promise((r) => setTimeout(r, 60));
const afterGc = c.debugRefCount().guestRefs;
check("dropped references are released", afterGc < beforeGc / 2, `${beforeGc} -> ${afterGc}`);
// ── 5b. Prototype pollution must not cross ────────────────────────────────
// JSON.parse is safe (it makes __proto__ an own property), but a membrane that
// assigns parsed keys onto a fresh object is not.
const before = ({} as any).polluted;
c.load(`__h.sink(JSON.parse('{"__proto__":{"polluted":true}}'));`);
check("guest cannot pollute host Object.prototype", ({} as any).polluted === before);
await c.endow("__p", { give: () => JSON.parse('{"__proto__":{"hostPolluted":true}}') });
await c.endow("__q", { show: (v: string) => check("host cannot pollute guest prototype", v === "undefined", v) });
c.load(`var got = __p.give(); __q.show(String(({}).hostPolluted));`);
// ── 6. Interrupt budget, and the VM survives it ───────────────────────────
// Direct: the budget on this call bounds it.
{
const t0 = Date.now();
try { c.load(`while (true) {}`, 100); } catch { /* expected */ }
check("runaway guest code is interrupted", Date.now() - t0 < 600, `took ${Date.now() - t0}ms`);
}
// Re-entrant: a guest function invoked BY THE HOST gets `callGuestRef`'s own
// budget, not the outer one. Budgets do not nest — see README "Limitations".
c.load(`globalThis.spin = function () { while (true) {} };`);
await c.endow("__s", { run: (fn: any) => { try { fn(); return "returned"; } catch { return "threw"; } } });
{
const t0 = Date.now();
try { c.load(`globalThis.__x = __s.run(spin);`, 50); } catch { /* expected */ }
const took = Date.now() - t0;
check("re-entrant call uses its OWN budget, not the outer one", took > 1000,
`took ${took}ms — if this is now <1s the nesting behaviour changed`);
}
c.load(`globalThis.__alive = 1 + 1;`);
check("VM usable after an interrupt", true);
// ── 7. Async, and clean teardown with a pending promise ───────────────────
await c.endow("__a", { never: () => new Promise(() => {}), soon: () => Promise.resolve("v") });
c.load(`globalThis.out = null; __a.soon().then(function (v) { out = v; });`);
await new Promise((r) => setTimeout(r, 30));
await c.endow("__o", { show: (v: string) => check("async host result reaches guest", v === '"v"', v) });
c.load(`__o.show(JSON.stringify(out));`);
c.load(`__a.never().then(function () {}, function () {});`); // deliberately unsettled
let disposeOk = true;
try { c.dispose(); } catch { disposeOk = false; }
check("dispose with a pending promise does not crash", disposeOk);
console.log(`\n${pass} passed, ${failures.length} failed`);
for (const f of failures.slice(0, 10)) console.log(" FAIL", f);
process.exit(failures.length === 0 ? 0 : 1);

81
third_party/pi/packages/agent/docs/mobile-handoff/README.md vendored Normal file
View File

@@ -0,0 +1,81 @@
# pi — design handoff
Work in numbered order. Each unit is self-contained and independently testable;
later units consume earlier ones.
```
01-harness/
01-delta/ op vocabulary, tracker, applier, codec [LANDED IN CHORD]
02-scopes/ storage scopes and list tags [STEP 1 ACTIONABLE]
03-execenv/ bounded Shell output, capture, spill [PRODUCTION CODE + TESTS]
04-tool-output/ the ToolOutput sink [SPEC ONLY]
05-assistant-output/ assistant partials, symmetric with 04 [SPEC ONLY]
02-plugins/
01-facets/ the facet system [SPEC ONLY]
02-sandbox/ isolated-vm membrane [CODE + 412 tests]
```
Base everything on a clean checkout of `origin/dev`.
## Read this first
**Three units ship working code. Four are specifications.** The table above says
which. Do not assume a doc describes something that exists.
**`01-delta/FINDINGS.md` is historical evidence, not an implementation queue.** Production lives in `packages/chord/src/delta/index.ts`: flush-time dirty tracking fixed D1, and production remeasurement closed D2. The explicit append/truncate API is rejected; see [`01-harness/01-delta/append-decision.md`](01-harness/01-delta/append-decision.md). The code beside the handoff remains prototype and benchmark evidence.
**If a doc and the code disagree, the code wins** — fix the doc and say so in the
commit.
**Port the tests before the implementation.** Each group's comment explains the
failure it guards against, and several of those failures are silent: wrong output,
no exception.
**Benchmark with `node --experimental-strip-types`, never through a transpiler.**
Measuring this module through `tsx` inflates results 2.6x. `FINDINGS.md` D5 lists
five more measurement traps, each of which produced a confident wrong conclusion.
## Unit status
| unit | ships | state |
| --- | --- | --- |
| **01-delta** | production implementation and tests in `packages/chord`; prototype evidence here | landed; D1 fixed, explicit text API rejected after production remeasurement |
| **02-scopes** | spec + [actionable Step 1 handoff](01-harness/02-scopes/implementation-handoff.md) + `scopes.variance.ts` | Step 1 scopes/list tags actionable, not implemented; JSONL Chord encoding/address interning deferred to separately approved Step 2 |
| **03-execenv** | production implementation in `packages/agent`; prototype evidence here | source-bounded adaptive output, lazy spill backpressure, and bash migration landed; bash's temporary checkpoint cadence moves to `ToolOutput` next |
| **04-tool-output** | spec + design notes | **not built.** The piece every measurement of the op encoding depends on |
| **05-assistant-output** | spec | not built. Same shape as 04; do it after |
| **02-plugins/01-facets** | spec, ~1800 lines | not built. §14 rewritten to match the sandbox PoC |
| **02-plugins/02-sandbox** | working PoC, 412 assertions | `npm install && npm run audit` |
## Suggested order
1. **`02-scopes` Step 1** — follow the [actionable implementation handoff](01-harness/02-scopes/implementation-handoff.md); stop for approval before its separate Step 2.
2. **`04-tool-output`** — reuse the landed adaptive publisher for generic tools, Chord event/durable batches, terminal flushes, and atomic memo checkpoints.
3. **05**, then **02-plugins**.
## Live bugs on `origin/dev`, independent of this design
- `drive/tools.ts:257` — `clearReplayCheckpoint` deletes `pendingToolOutput` before
re-executing a replay-safe tool. Memos exist so a replayed tool skips work, and
skipped work emits nothing, so output for memoised work is lost today. Seed from
it instead (`harness-tools.md` §7.4).
- `runtime/progress.ts:44` — `commitWrite(item)` captures the value at call time
and writes fire-and-forget, so an older checkpoint can land after a newer one.
- memo and checkpoint are two transactions (`drive/tools.ts:112` vs
`progress.ts:44`). They must be one (`harness-tools.md` §7.5).
**Expected test churn:** nine tests assert the old cleanup write set and will fail
once `retireScope` replaces the per-address deletes. That is the change landing.
## Environment
- Node 22+ for every shipped `.ts` file. They run under
`node --experimental-strip-types` with no build step and no dependencies.
- In the pi repo, tests run from the package:
`cd packages/agent && npx vitest run --config vitest.harness.config.ts`.
The root vitest config does **not** alias `@earendil-works/pi-ai`; the
per-package harness config does.
- Typecheck with `npx tsgo --noEmit` from the repo root. **Baseline is ~788
pre-existing errors**, almost all in `packages/ai/test`. Count only:
`grep "error TS" | grep -E "packages/(agent|session-backends)/src"`.
- `packages/ai` cannot be built offline — model data is fetched at build time.

1158
third_party/pi/packages/agent/docs/plugins.md vendored Normal file
View File

File diff suppressed because it is too large Load Diff

254
third_party/pi/packages/agent/docs/post-wp05-roadmap.md vendored Normal file
View File

@@ -0,0 +1,254 @@
# Post-WP05 roadmap audit
**Audit baseline:** `5507d76ee` (`dev`, 2026-08-27).
**Status:** Planning inventory, not a behavior contract. [`harness.md`](harness.md) remains normative where it agrees with the current product boundary. Contradictions listed below must be resolved explicitly; this document does not silently choose one side.
## Scope and method
This audit covers the durable AgentHarness and its directly coupled Session backends and presentation path:
- `packages/agent/src/harness`, `packages/agent/src/search`, and their tests/docs;
- `packages/session-backends/sqlite-node`;
- the current `packages/protocol`, `packages/client`, and `packages/server` presentation path;
- `packages/coding-agent/src/experimental` and its focused tests where they host that path;
- telemetry plumbing in `packages/telemetry`, `packages/ai`, and `packages/agent`.
The inventory was checked against current source, tests, package READMEs, the complete `harness.md`, completed WP00–WP07 status, the actionable WP08 handoff, explicit stubs/TODOs/skips, and the shipped package boundaries. Historical handoffs are not treated as backlog when current source and the completed WP05 contract supersede them.
## Executive result
WP05 is complete through M10. Its remaining assistant-output work is owned by the [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md) and its numbered prerequisites. The current Harness execution graph has no unfinished runtime path: `watchSession()` is the sole `SliceNotImplemented` Harness method.
That does **not** mean the surrounding durable system is complete. The remaining audit findings are:
1. normative JSONL snapshot-compaction behavior with no implementation;
2. one required Harness method stub (`watchSession`);
3. a deliberate removal of raw `RemoteSession` that conflicts with later normative WP06/`harness.md` text;
4. a public search type skeleton that conflicts with the newer search design and has no implementation;
5. a full telemetry vocabulary whose only production span is the tool-hook span;
6. smaller repository, client-watch, query-bound, documentation, and end-to-end-test gaps.
WP07 completed SQLite host-ownership alignment and live-source fork support after the audit baseline; its historical handoff is [`work-packages/07-sqlite-host-ownership-live-forks.md`](work-packages/07-sqlite-host-ownership-live-forks.md). WP08 now owns the separate named-branch, tree-state, and bounded-memory fork redesign; its actionable handoff is [`work-packages/08-named-branch-streaming-forks.md`](work-packages/08-named-branch-streaming-forks.md).
## Required missing functionality and contract contradictions
### R12 — Session-wide Harness watch
**Evidence**
- `AgentHarness.watchSession(context)` is public in `src/harness/agent-harness.ts`.
- `Harness.watchSession()` throws `SliceNotImplemented("watchSession")` in `src/harness/runtime/harness.ts`.
- `SessionSnapshot` currently contains only `{ lanes: LaneInfo[]; faulted: boolean }`.
- Lane watch, event buffering, delivery-tail barriers, and `resnapshot()` already exist in `src/harness/events.ts` and `src/harness/runtime/lane.ts`.
**Remaining boundary**
Define one coherent capture and fold for dynamic lane inventory and fault state. Decide whether the intentionally small `SessionSnapshot` stays small or gains session metadata/stats/global configuration. Then implement snapshot-before-events, lane creation, resnapshot, listener reentrancy, close/fault behavior, and a session reducer if event-only replication is promised.
**Dependency**
Independent of the mobile assistant-output handoff and SQLite internals. It should precede any revisioned Transcript service or remote session-wide observation built on it.
### JSONL snapshot compaction
**Evidence**
`harness.md` §1.7 normatively specifies temp-file-and-rename snapshot compaction, preserved sequence high-water marks/list cursors, threshold checks on open, and reclamation after terminal/outcome deletions. `JsonlStorage` implements atomic creation, torn-tail repair, legacy-v3 rewrite, append, and fork snapshots, but no current-state snapshot rewrite or dead-byte accounting exists.
**Consequence**
Superseded `pi.op.state`, deleted pending payloads, deleted tool checkpoints, and deleted assistant-frame lists remain physical bytes indefinitely. Generic compaction and the [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md) are complementary. Compaction reclaims dead session-scoped write history after the fact; the handoff moves pending assistant/tool output into ephemeral scopes so it never becomes main-log history, and replaces full/per-frame replication with Chord op batches.
**Dependency**
The assistant-output handoff does not depend on J1: scoped storage is the intended lifetime mechanism for pending output, while J1 remains the reclamation mechanism for superseded session-scoped state. Measure both independently so their effects are not conflated.
### Remote Session contract contradiction — decision required
**Current product boundary**
Commit `f8a6e670d` deliberately deleted `RemoteSession`, its raw Session RPC protocol, and the server mutation-scope manager, replacing them with attachment-fenced routed semantic services. Current protocol/server READMEs explicitly state that real `Session` and `AgentHarness` objects remain process-local. The shipped path supports Session discovery/creation/attachment, main-lane prompt/watch, and allowlisted plugin-service calls; it does not expose `Session`, `SessionMutation`, values/lists, branches, or storage over RPC.
**Conflicting contract**
WP06, written after that deletion, requires the “current keyless RemoteSession mutation transport,” includes remote begin/read/commit/publication/end in required tests and stop conditions, and forbids its removal. `harness.md` §§2.8 and 9.1 likewise state that local and remote implementations preserve that lifecycle. No such implementation, protocol schema, client facade, server-held scope, worker adapter, or conformance test exists at the audit baseline.
**Required decision**
Choose one before scheduling implementation:
1. **Process-local Session remains intentional:** remove the false RemoteSession requirements from normative current-state docs while preserving semantic service RPC; or
2. **RemoteSession is required:** commission a dedicated package for mutation begin/read/commit/end, disconnect/timeout cleanup, publication-before-end, values/lists/branches/entries/stats, protocol validation, client/server/worker adapters, and remote conformance.
Do not count the current lane-watch compatibility RPC or plugin-service RPC as RemoteSession. Do not restore the deleted 507-line facade unchanged: it predates the keyless Session/Branch contract and relied heavily on untyped decoding.
### Telemetry contract exceeds implementation
**Evidence**
`src/harness/telemetry.ts` and generated `docs/telemetry-schema.md` declare `pi.ai.request`, operation, checkpoint, turn, step, tool, hook, sleep, event-handler, and session-write spans. Production source starts only `pi.harness.hook`, and only for registered `before_tool`/`after_tool` handlers. AI options propagate `telemetryContext`, but no provider path starts `pi.ai.request`. Server request ingress has cancellation but no trace carrier or client/server RPC spans. `TODO_CONTEXT` remains at transport/worker lifecycle and event-delivery boundaries.
**Remaining boundary**
Treat this as separate packages:
1. local Harness/Session/AI instrumentation and runtime tests;
2. RPC trace-carrier/client/server propagation;
3. an optional application-selected exporter/adapter.
First reconcile whether every declared span is still wanted. If retained, implement it; if not, remove the unsupported public schema surface and correct `harness.md`. Do not mix telemetry with Context/RPC cancellation, which already has independent request-ID signaling.
### S3 — Search
**Evidence**
`src/search/index.ts` exports a public `SessionSearchService` skeleton with `sync()`, `notify()`, and array-returning `searchEntries()`. `harness.md` §2.8 instead specifies a standalone service, separate catch-up/notify utilities, generation-aware cursors, and optional `AsyncIterable` entry search. There is no factory, sync utility, cursor store, projection, or source SQLite FTS implementation. At the audit baseline the SQLite README advertised nonexistent `createSqliteSessionSearch()` behavior; this audit corrected that README rather than treating the absent API as implemented.
**Remaining boundary**
Before implementation, replace or reconcile the draft public interface and decide metadata filtering (`cwd`), candidate restriction, or indexed metadata. Post-filtering after ranked `limit` is unsound. Then implement standalone catch-up and a separate SQLite FTS5 projection; do not add repository search methods.
### R11 — Schema migrations, activation-gated
No current format-4 migration is required. Memory is current-only; JSONL and SQLite reject unsupported storage versions; SQLite runs only idempotent `001_initial.sql`. R11 becomes required immediately before the first incompatible durable storage version/address/state change after format 4 is stabilized. It is not prerequisite work for the mobile assistant-output handoff or current WIP format replacement.
When activated, it must provide ordered transactional migrate-on-open under exclusive ownership, version-specific JSONL decoding plus post-migration compaction, and total mappings for every reachable open operation leaf and surviving value/list.
## Correctness and data-safety debt
### SQLite host ownership and live-source forks — completed by WP07
The authoritative product rule is in `plugins.md`: exactly one host-assigned process owns writable Session authority; normally it is the Session worker, while the server may temporarily own a newly created or forked destination before closing it and handing it off. Storage backends do not implement writer ownership. The server closes a worker before destructive repository administration.
SQLite now follows that rule: the writer-lease schema/module, claims, renewal timer, lease-loss path, and pre-commit callback are gone, with no replacement lock or ownership primitive. Create/open/fork/delete retain repository-local ID reservation. Metadata open and deletion use a true no-create read-write mode; listing and external fork sources use no-create read-only connections.
Same-repository forks retain the source `commitQueue` ordering seam. A source owned elsewhere, including a live worker, is read from its exact canonical container through one independent read-only deferred WAL transaction. Focused per-file and shared-container tests commit a complete later source transaction after the reader establishes its snapshot and before it closes: the first fork excludes the transaction wholly and a later fork includes it wholly.
WP07 also completed canonical `(containerPath, sessionId)` active identity, safe explicit-ID filenames, custom `databasePath` parent creation, Session-scoped shared deletion, WAL/SHM cleanup, and all-settled SQLite repository close. Writable open/delete reject foreign metadata; foreign fork sources are read only from their exact path. Equal-`createdAt` list ordering still has no deterministic tie-break and remains a later behavior-preserving cleanup.
### Repository close ownership is unspecified
`JsonlSessionRepo.close()` contains the only active Agent source TODO and closes no open Session handles. Memory still uses fail-fast `Promise.all`; SQLite now performs backend-local all-settled cleanup of currently open handles, but an already-admitted create/open/fork can still register a handle after repository close captured that set. `SessionRepo` itself declares no `close()` method and shared conformance does not define repository-to-handle ownership or draining of admitted repository operations. Resolve ownership and common cleanup in one repository-lifecycle package; do not patch one backend further without deciding the common contract.
### Client watch staleness after disconnect
`Client` clears its active watch-listener map on disconnect, but existing `LaneWatch` objects keep local `ready`/`started` state and old watch IDs. After reconnect/reattach they can call `start()` or `resnapshot()` and fail remotely instead of deterministically rejecting as stale, contrary to `packages/client/README.md`. Service-subscription objects have the sibling problem: their listeners are cleared and surviving objects become silently dead. Fix both with connection/attachment incarnation fencing and focused reconnect tests. This is independent of R12: the current client method is a compatibility main-lane watch.
### Query bounds and SQLite bind limits
- SQLite `getEntries(ids)` emits one placeholder per requested ID and can exceed the engine’s variable limit.
- Entry, usage, and branch limits use ad hoc `Math.max(0, limit)` behavior. Memory and SQLite diverge for `NaN`, infinities, fractions, and extreme values; unlike list reads, there is no shared normalization contract.
Define cross-backend query-limit semantics in agent conformance, then chunk SQLite ID lookups. This is a storage-contract hardening package, not part of WP07.
### Harness contract and conformance closure
- The public `OperationStatus` includes `"running"`, but lane inspection, snapshots, and `reduceLaneSnapshot` currently produce only `"open"` or `"aborting"`. Define and implement its producer or remove the dead variant.
- The pre-rewrite abort contract bound/published `operation_abort` before resolving the cancellation promise and signalling the live gate. Current `Lane.command()` materializes the result — resolving/signalling — before constructing and binding the event batch, although it still binds recipients before releasing the Session mutation line. Decide whether to change the implementation or retain/document the current no-interleaving order; add an explicit ordering test.
- The production gate-close contract permits only `HarnessClosed | HarnessFault`, but the private source primitive accepts any `Error` and isolated tests use that wider type. Narrow the source declaration and fixtures or explicitly retain the private widening.
- Part 9 of `harness.md` is the required conformance matrix. Existing focused tests cover the graph extensively, including cancellation reconciliation over all 13 leaves, but there is no audited one-to-one proof that every close/reopen leaf case and every race row has both deterministic orders. Audit the matrix and add only the missing cases rather than claiming blanket completion.
Keep this package separate from telemetry, RemoteSession, and the mobile assistant-output handoff; it is local contract/test closure.
### Disabled real worker persistence regression
`packages/coding-agent/test/experimental-remote-runtime.test.ts` still skips “completes and persists a prompt through the worker-owned Harness” with the obsolete note “Re-enable with runtime no-tool execution.” No-tool execution now exists. Re-enable or replace it with a deterministic faux-provider real-worker persistence test; do not use a real paid provider.
## Performance debt
### Mobile assistant-output handoff — durable and replication amplification
The user-supplied motivating mini Session outside the repository is 303,920 bytes across 569 physical lines. These external measurements are evidence, not a reproducible checked-in fixture:
- 477 assistant-frame appends totaling about 118,418 serialized write bytes;
- 12 frame-list deletes; physical lines mentioning the frame namespace total 148,214 bytes;
- about 51,568 bytes of superseded `pi.op.state` writes and 26,192 bytes of one structural preparation, showing why generic JSONL compaction and frame-specific bounding are distinct.
The authoritative design is the [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md), following the numbered `01-harness` prerequisites in [`mobile-handoff/README.md`](mobile-handoff/README.md). Chord delta tracking has landed; scoped storage, tool-output integration, and assistant-output integration have not.
Implementation must add a deterministic repository fixture and measurement script that reproduces or replaces these figures, then measure Memory logical elements, SQLite rows/pages/WAL, JSONL peak sidecar/main-log bytes, and reopen/replay time. The handoff must preserve unknown-outcome recovery, invocation fencing, non-blocking provider streaming, and settlement retirement while eliminating per-frame durable writes and quadratic `message_update` replication. Numerical budgets belong in its implementation handoff/tests, not in a competing standalone design.
### SQLite branch divergence
`createDivergentBranchForEntry()` copies every row after the newest compaction; with no compaction it copies root-through-parent. A first divergence from a long uncompacted transcript is therefore O(history) writes. This exposes an internal contradiction in `harness.md` §2.6: its opening bounded-prefix promise conflicts with its own compaction-based copy algorithm, which the implementation follows. Change both the specification and segment representation so a divergence can reference a covering segment at the parent boundary. Preserve shared-container support and add large uncompacted divergence plus chain-soundness tests/benchmarks.
### Fork contract and materialization — WP08 actionable
All current backends materialize source-sized fork snapshots. Branch scope defaults to `main` without named-Branch ancestry validation; tree scope omits application values/lists; JSONL closed-source fork replay may repair its source. WP08 replaces that contract with required named-branch or tree scope, one closed built-in-state policy, complete current application state for tree forks, and backend-specific bounded-memory copy procedures. It preserves WP07's host ownership, physical identity, same-repository ordering, and independent live-source WAL boundary.
### SQLite catalog, statements, stats, and reclamation
- Default per-session `list()` synchronously opens/configures every SQLite file serially and silently skips failures. A shared container or external catalog is the scalable deployment choice; bounded async scheduling does not make `DatabaseSync` nonblocking.
- Most hot queries prepare a fresh statement each call. Cache narrowly owned statements after measuring.
- Each usage row parses and rewrites the full aggregate JSON usage payload.
- Shared-container row deletion does not reclaim pages. Define maintenance/VACUUM policy separately; never add `VACUUM` to ordinary deletion casually.
These are measurable optimization/operations packages, not correctness fixes.
### Pending-payload amplification and mutation-line parallelism
Queued payloads deliberately write `pendingEntry → immutable entry`; measure pathological payloads before changing it. Keyed Session mutation lines remain optional and require profiling plus a fresh mutable-ownership audit. Neither is current correctness work.
## Behavior-preserving cleanup and documentation repair
Completed by this audit:
- rewrote the SQLite README’s nonexistent `SqliteSessionRepository`/search APIs, wrong `await using` and `appendMessage` example, FTS trigger/rebuild claims, and “one shared connection” claim;
- reconciled `harness.md`’s default one-file SQLite wording with the supported optional shared container;
- reconciled `harness.md` Part 8 with this audit and corrected stale drive-ownership/RPC-cancellation status in `telemetry.md`;
- corrected coding-agent settings docs: install telemetry configuration also controls selected provider attribution headers.
Remaining cleanup:
- **Harness-owned DTO boundary:** the durable Harness still imports legacy agent-loop DTOs from `src/types.ts`: `AgentMessage`, `AgentToolResult`, `AgentToolCall`, `AgentTool`, `QueueMode`, and `ThinkingLevel`. Make a later one-shot cut to independent `HarnessMessage`, `CustomHarnessMessages`, `HarnessToolResult`, `HarnessToolCall`, `HarnessQueueMode`, and `HarnessThinkingLevel` definitions; keep `AgentHarnessTool` as the executable Harness configuration type. Update Harness internals, root exports, declaration-merging tests, documentation, and experimental coding-agent consumers together. Do not alias the new message/tool DTOs back to legacy types, because that preserves the coupling this work is intended to remove.
- Preserve shared-container support; do not remove it incidentally.
- Remove or use unused `insertEntryRow()` and `insertUsageLedgerRow()`.
- Consolidate duplicated SQLite branch payload/structure scan plumbing only after correctness tests pin both paths.
- Decide whether unused `sessions.metadata` and unmeasured indexes have a future owner before schema removal; do not change schema casually.
- Consolidate `startAiSpan()`/`startHarnessSpan()` implementation only if the telemetry surface is retained.
- Mark historical durability documents as delivered where they still read as implementation queues. WP00–WP07, `runtime-simplification.md`, `values.md`’s old consumer deferrals, and external-finalization designs are not active runtime backlog.
## Optional or deferred product capabilities
These are not blockers for the durable Harness:
- standalone S3 search after its API decisions;
- Accounts removal and revisioned Transcript production;
- authenticated workspace/client authorization for the experimental local server;
- private returned references, service flow control, multi-pane presentation, plugin kernel/reload completion, and version-skew negotiation;
- administrative precise rewrite tooling;
- a partitioned Postgres backend/retention policy;
- generic remote Harness/object capabilities;
- DeltaState or delta replication before concrete snapshot pressure justifies it;
- a production telemetry exporter;
- schema migrations before an incompatible stabilized-format change activates R11.
## Recommended dependency order
The order is by data safety first, then dependencies. Independent tracks may proceed in parallel only when they do not edit the same contracts.
1. **Harness contract/conformance closure.** Resolve `OperationStatus.running`, abort signal/event binding order, gate-close typing, and the Part 9 coverage matrix.
2. **Remote Session decision (decision only).** Resolve the false normative boundary early. If process-local wins, repair the docs. If raw RemoteSession wins, later create a dedicated protocol/client/server/worker package; do not fold it into telemetry or R12.
3. **Client watch/subscription staleness** and **repository lifecycle contract.** Small independent correctness packages; complete them before expanding server/worker lifecycle semantics. The lifecycle package must also address Memory's fail-fast repository close.
4. **[Mobile Harness handoff](mobile-handoff/README.md).** Follow its numbered prerequisites through scoped storage, tool output, and assistant output; preserve all recovery boundaries and land deterministic amplification measurements.
5. **JSONL snapshot compaction.** Implement the already-normative physical reclamation path and metrics for remaining session-scoped history.
6. **R12 session-wide watch.** Complete the only Harness method stub before building revisioned Transcript/session-wide remote observation.
7. **Telemetry, if retained:** reconcile schemas, then local instrumentation, then RPC propagation, then an optional exporter. RPC propagation follows the Remote Session/product-boundary decision.
8. **WP08 — named-branch and streaming forks.** Implement the actionable handoff without reopening WP07 ownership or lifecycle decisions.
9. **SQLite branch/query performance hardening.** Keep separate from completed WP07 ownership alignment and WP08 fork semantics; require benchmarks.
10. **S3 search.** Resolve its API/filter/cursor decisions, then implement catch-up and the standalone FTS projection.
11. **R11 migrations.** Activate immediately before the first incompatible stabilized durable schema change, not earlier.
## Stop conditions for roadmap accuracy
This inventory must be updated when any of these facts changes:
- `watchSession` stops being the sole Harness `SliceNotImplemented` method;
- raw RemoteSession is either recommissioned or removed from the normative contract;
- JSONL snapshot compaction lands;
- telemetry schemas are implemented or removed;
- S3’s public API is reconciled;
- a durable format change activates R11;
- WP08 lands or its fork contract changes;
- the host-authority contract changes.

215
third_party/pi/packages/agent/docs/rpc.md vendored Normal file
View File

@@ -0,0 +1,215 @@
# Facet Service RPC
Chord owns the application-neutral service semantics and pluggable strict-JSON connection
boundary. Pi owns the concrete wire envelope, routing, attachment state, and error adapters
described here. The current implementation treats `JsonValue` as a static contract and defers
runtime rejection of unsupported values to the concrete serializer.
> **Status:** Design specification for experimental facet-service RPC semantics.
## Role
`provide()`/`use()` and `provideMany().spawn()`/`observe()` are the facet system's hidden RPC. Facets share TypeScript service contracts, while the transport carries service/member identifiers, strict JSON values, request/subscription correlation, keyed generations, and binding control messages. Hosts construct the typed local implementation or facade; TypeScript types and arbitrary objects never cross the wire. Independently loaded processes may temporarily run different source generations, so their service contracts must remain forward compatible across the supported skew window; version negotiation remains deferred.
```text
session/server facet: provide() / provideMany().spawn()
↕ hidden service RPC
presentation/session facet: use() / observe()
```
The service system is the extension boundary. Presentation facets receive semantic services and replicated state; they never receive a raw Harness, Session, tool registry, hook registry, credential store, or storage handle.
## Non-goals
Do not serialize `Context`, `AbortSignal`, telemetry objects, callbacks, tools, hooks, functions, or arbitrary object graphs. Do not make core Harness or Session implementations aware of transport mechanics. Do not make disconnect perform durable or service-owned cancellation. Do not build per-method codecs for values already constrained to JSON.
## Service contracts and typed facades
A service token is a shared TypeScript contract and stable service ID. It is not a generated descriptor and creates no provider. Tokens are remotely publishable by default; process-local tokens declare `{ local: true }`. `provide()` adds one singleton implementation to the host graph. `provideMany()` registers one multi-instance service owner during facet setup and returns a `ServiceSpawner` whose later `spawn()` calls publish instances. The host automatically publishes every non-local provision. A token has one mode in one host service graph: mixing singleton and keyed use is an error.
The provider classifies every exposed implementation member as a method or Chord-created `ReplicatedState` and publishes that member table in subscription snapshots. The consumer obtains member names from ordinary property access—for example, a JavaScript `Proxy` receives `"state"` for `models.state` and `"refresh"` for `models.refresh(context)`. Accessed slots are validated against the provider-announced kind when the facade binds.
Local and remote `use()` both return a stable, lazy typed facade shared by consumers of that token. During synchronous facet setup the facade is disconnected, so setup can capture it but cannot invoke methods, read state, or register member subscriptions. After assembly, a local facade resolves through a direct process-local implementation slot and a remote facade binds through the host's connected services. Reloading the providing facet temporarily marks that same facade unavailable, then swaps its target; RPC singletons clear readiness and install a complete replacement snapshot on their existing subscription so captured methods and member facades address the replacement. While no provider is bound, invoking a method fails and state remains unhydrated; no call is queued merely because it was made through a proxy.
Remote methods return promises and accept and return strict JSON apart from their declared `Context`; `void` is a successful response without a result field. Private returned references are not supported. The client removes the context before transport and the receiving host constructs a fresh local context. The contract position is host-controlled and must be consistent; the examples use one required trailing `Context`. Business absence is JSON `null` or an options object, never transported `undefined`.
Use static assertions and runtime validation. Static checks constrain remote methods and replicated-state members; runtime boundaries reject unsupported members and non-JSON arguments, results, and state values. TypeScript supplies typed facades but does not authenticate a peer or create runtime metadata.
`{ local: true }` removes only remote publication and its wire-contract restrictions. Local and non-local provisions otherwise use the same dependency ledger, activation order, stable singleton slots, keyed-instance generations, observer cancellation, disposal, and provider-facet reload. Local singleton slots and the local keyed registry hold arbitrary object contracts directly; non-local provisions additionally install their implementations in the remote provider.
## Dependency ledger
Type erasure does not hide service identity: every service token retains its stable ID at runtime. Facet environments are created by the host with a non-forgeable owner identity, and their setup-time service methods append to a generation-scoped ledger:
- `provide()` records a singleton provision;
- `provideMany()` records a keyed provision;
- `use()` records a singleton requirement; and
- `observe()` records a keyed requirement.
Facets always call unqualified `env.use()` or `env.observe()`; routing is not encoded in the call. These operations return source-independent disconnected handles during setup. After setup, each connection returns its provider-generated service catalogue, and the host binds every requirement to its local provision or exactly one connected provider. The method supplies the mode and the token supplies the ID. The host therefore needs no reflection over the erased `T` and no handwritten parallel dependency list.
First acquisition or provision is permitted only during facet setup. Later commands, hooks, and activation callbacks use setup-acquired singleton facades, observer registrations, or `ServiceSpawner` capabilities. In particular, dynamic instances are spawned through the capability returned by `provideMany()`; late spawning cannot introduce a previously undeclared provision.
After setup, each host privately resolves the recorded requirements against local provisions and connection catalogues to reject missing providers, duplicate remote offers, and mode mismatches and to derive lifecycle edges. A selected-Session connection with no live attachment may provisionally accept unresolved requirements as unavailable; attachment validates them against the worker's generated catalogue and caches that catalogue for later detached generations. Connection bindings are generation-owned and include only selected requirements, so failed or retired generations release their subscriptions without disposing the underlying transport connection. This internal service graph is distinct from the module loader's source import graph; it is not a facet-authored or facet-visible plan.
## Bindings and identity
A presentation host combines services from its connected server and selected Session in one graph. All of its facets use the same unqualified environment API. A Session-service call never accepts a client-selected durable `sessionId`; the server authorizes and routes the presentation's selected Session binding to its worker.
### Server control plane
Session listing and management are ordinary server singleton services, not generic remote `Session` methods. `SessionDirectory` exposes presentation-safe session summaries as replicated state. `SessionManagement` exposes `create`, `remove`, `attach`, and `detach` methods. A TUI or web facet consumes both through its environment:
```ts
const directory = env.use(SessionDirectory);
const management = env.use(SessionManagement);
```
The server binds one service provider to each presentation connection. It derives workspace and client authority from locally authenticated connection identity, not from summary fields or method arguments. It may project directory state per client; either way, summaries never expose server-private fields such as owner IDs or working directories.
`management.attach(sessionId, context)` changes the selected-Session services in the presentation host. The server closes the presentation's previous Session-scoped requests, subscriptions, and observer tasks, binds the presentation's Session services to the worker, then hydrates their singleton state and keyed-instance directory. The server authorizes the selected Session against the connection identity. Attachment state is host control state reporting this selection and its health; it is not a directory service. `detach()` performs the same cleanup without a replacement.
The host needs a private, host-owned binding incarnation for that route. It changes when the presentation attaches, detaches, switches session, or replaces a failed worker. Its representation is deliberately unspecified. The binding prevents a delayed frame for the old selected session from being applied to the new one; it is not a facet-visible service value or a substitute for authorization.
A replicated-state source has structural identity:
```text
(provider binding, service ID, optional instance key + generation, member name)
```
There is no separately discoverable state ID. An added instance key is an application-level logical key. Its host-owned generation changes when a closed key is reused, so a stale proxy cannot call the replacement. `requestId` identifies one transport invocation for response and cancellation. A Harness/tool `invocationId` may be a useful instance key—as in the question example—but it does not replace the service, binding, or generation parts of the live address.
## Calls, context, and routing
A call carries enough control-plane information to select a provider binding, service, optional keyed instance, and member, plus a request ID, JSON arguments, and trace carrier. The server may parse those control-plane fields to route a Session call, but it does not parse facet business payloads or load facet contracts. The service endpoint validates the member and values, creates a request-local abort controller and `Context`, installs authenticated identity, and invokes the local implementation.
The client maps `context.abortSignal` to cancellation of that one request. Disconnect cancels that connection's active calls and closes its subscriptions. Neither action cancels service-owned work or writes durable Harness cancellation. Per-client request correlation reaches the worker so request IDs from different presentations cannot collide.
## Replicated state and keyed instances
`ReplicatedState` is authoritative latest-value replication, not event history, durable storage, a CRDT, or multi-writer state. A cold replica has `value === undefined`; subscribing before hydration records a listener without invoking it. Hydration installs a complete snapshot atomically before later updates are delivered, so there is no snapshot/update gap. Once hydrated, `value` is synchronous and subscribing reports the current value followed by later updates. The first snapshot callback uses a fresh hydration context; an already hydrated replica uses a fresh local delivery context rather than retaining the original write context. State values are borrowed immutable JSON and are not defensively cloned; callers must not mutate or retain them.
Remote hydration uses a fresh delivery context parented to the subscription, while updates reconstruct fresh delivery contexts from source trace metadata. Disconnect, provider withdrawal, replacement, and route switching clear readiness. Reconnect or replacement installs a complete snapshot before later updates. The transport buffers updates that race hydration and checks their sequence. Acknowledgements, flow control, and gap recovery remain separate protocol mechanics.
`observe()` is keyed-instance discovery, not a `ReplicatedState` containing proxies. It reconciles a complete initial directory with ordered additions, replacements, and removals. Each instance's initial state members hydrate before its observer task starts. Closing an instance rejects new calls, aborts only that instance's observer task, and allows admitted calls to settle. A Session facet's `env.observe()` registration aborts old tasks when that facet generation closes; the replacement generation reconciles the fresh directory.
## Private returned references
Private returned references are outside the initial service contract. Prefer keyed services for discoverable live instances. If a concrete feature requires caller-private remote identity, its reference must be passed explicitly rather than discovered by `observe()` and scoped to the recipient and provider binding.
No generic Harness projection is part of this design. Raw Harness, Session, lane, tool, hook, and storage objects remain local authority. If a future integration needs a remote callback or a general object capability, it needs a separate explicit protocol and policy; it is not an extension of service RPC.
## Context, cancellation, and telemetry
Every remote method receives a fresh local `Context`; the sender's object, signal, telemetry implementation, and arbitrary typed values never cross the wire. The client maps the call's abort signal to that request and injects a trace carrier. The endpoint constructs a request-local abort signal and telemetry parent. Cancellation is forwarded through the server to Session workers and remains isolated by client plus request ID.
The span relationship is:
```text
caller
└─ rpc.client
└─ rpc.server
└─ service implementation
```
Three cancellation domains remain separate: aborting one RPC invocation; explicitly cancelling service-owned work such as `job.cancel()`; and durable Harness cancellation such as `requestAbort()`. Transport cancellation and disconnect perform only the first. Work that outlives a call must detach into a service-owned task with its own controller and telemetry root.
## Security and lifecycle
Only loaded service tokens not marked local may be registered at the remote boundary. Only the owning `ServiceSpawner` may spawn instances. Services marked `{ local: true }` are never discoverable remotely. Local services may use unrestricted object contracts. Remote providers validate member kinds, while concrete serializers enforce JSON business values. Clients cannot forge control envelopes as ordinary values, choose instance generations, select a different Session route in a service call, or cancel another client's request.
The server authenticates connections, authorizes attachment, and reconstructs client identity in the service `Context`. Ordinary business arguments never carry authority. Credentials, prompts, completions, tool data, filesystem contents, and other sensitive values require an explicit presentation-safe contract.
Facet environments own registrations, added service instances, observations, and resources explicitly registered through `own()`. Connection bindings own their transport subscriptions and active request controllers. Already-admitted inbound calls are not attached to the providing facet lifecycle: provider withdrawal rejects new calls, but an admitted method may continue while the old facet deactivates. The provider's own Session work remains alive unless its lifecycle policy stops it.
## Tests
The facet-facing semantics are tested over loopback and framed transports:
- setup-time dependency-ledger ownership, rejection of late acquisition, local and remote `use()`, keyed-provider ownership, singleton/keyed mode validation, token-driven RPC publication, lazy member access, stable local and RPC singleton facades across provider-facet replacement, and `{ local: true }` services remaining unreachable remotely;
- strict JSON boundaries, method context reconstruction, and request cancellation isolation without serializing context values;
- server/Session facet isolation, selected-Session switching, stale-frame rejection, and worker-side per-client request correlation;
- cold and hydrated `ReplicatedState`, snapshot/update race freedom, fresh delivery contexts, and clearing/re-hydration on disconnect, reconnect, and provider replacement;
- instance directory hydration, ordered reconciliation, state hydration before observer tasks, generation-based stale rejection, and task cleanup on close or switch.
Additional tests cover authenticated attachment and identity, telemetry propagation, flow control and gap recovery, plus the question and shared-review application patterns. If private references are added, they require separate lifetime and isolation coverage.
## Open protocol mechanics
Exact service call, cancellation, subscription, snapshot/update, keyed-instance, unavailable, and replacement frames are defined in `packages/protocol/src/protocol.ts`. Provider and namespace layers own member classification, lazy facades, buffering, and sequencing.
Still open are acknowledgements, flow control, sequence-gap recovery, reference collection if references are added, protocol-version negotiation, and how a future multi-pane presentation represents more than one selected Session. Singleton provider replacement must continue to install a complete replacement snapshot on the existing subscription so method and state member slots retain identity.
## Example: directory and selected session
The directory and management services are normal server services. Their contracts carry only presentation-safe values:
```ts
interface SessionSummary {
serverId: string;
sessionId: string;
createdAt: string;
}
interface SessionDirectory {
readonly state: ReplicatedState<{ revision: number; sessions: SessionSummary[] }>;
}
interface SessionManagement {
create(options: { id?: string }, context: Context): Promise<SessionSummary>;
remove(sessionId: string, context: Context): Promise<void>;
attach(sessionId: string, context: Context): Promise<void>;
detach(context: Context): Promise<void>;
}
const SessionDirectory = defineService<SessionDirectory>("pi.session-directory");
const SessionManagement = defineService<SessionManagement>("pi.session-management");
```
A server facet derives the client from an authenticated `Context`, authorizes the requested Session, and performs the binding transition:
```ts
serverContext.provide(SessionDirectory, { state: directoryState });
serverContext.provide(SessionManagement, {
async attach(sessionId, context) {
const client = requireClientIdentity(context);
authorizeSession(client, sessionId);
await attachments.bind(client.clientId, sessionId, context);
},
async detach(context) {
await attachments.unbind(requireClientIdentity(context).clientId, context);
},
});
```
A presentation facet renders and selects Sessions:
```ts
setup(env) {
const directory = env.use(SessionDirectory);
const management = env.use(SessionManagement);
const tui = env.use(Tui);
tui.commands.register("sessions.switch", async (operation) => {
const snapshot = directory.state.value;
if (snapshot === undefined) return;
const sessionId = await tui.select(
"Sessions",
snapshot.sessions.map((session) => ({ label: session.sessionId, value: session.sessionId })),
{ signal: operation.abortSignal },
);
if (sessionId !== undefined) await management.attach(sessionId, operation);
});
}
```
Another facet in the same presentation acquires Session services through the same API:
```ts
setup(env) {
const models = env.use(Models);
// After attach() settles, `models` addresses the selected worker.
}
```
The presentation never routes `models` with a selected `sessionId`; its host routes each service token and transport retains the selected-Session binding. The server closes the prior Session binding's resources before hydrating the new one.

391
third_party/pi/packages/agent/docs/runtime-simplification.md vendored Normal file
View File

@@ -0,0 +1,391 @@
# Existing AgentHarness runtime simplification
## Scope
Rework the real implementation under `packages/agent/src/harness/runtime/` and the canonical durable types in `packages/agent/src/harness/session/types.ts`. This is not the isolated scratch spike.
The execution graph is total and public drive is enabled. Format 4 is still work in progress, so the durable type replacement required no migration or compatibility representation.
## Implementation status
Pre-M6 completion measurements:
- pre-simplification runtime at `eb1185d93`: 5,358 TypeScript lines;
- runtime after the first simplification at `417905647`: 4,667 lines;
- simplified substrate at `0e77e57d9`: 4,654 lines;
- total pre-M6 reduction: 704 lines (13.1%).
Implemented by the simplification pass:
- explicit `ContinueOperationResult<T>` with `cancel_requested` instead of implicit `undefined`;
- concrete generation, deferred-poll, and tool-call phases: prepare immutable input, publish durable intent, perform the effect, publish the durable outcome;
- shared assistant stream protocol consumption and Drive response lifecycle;
- `publishResponse` for common assistant/deferred durable publication and `publishConfigurationFailure` for their identical pre-intent failure transition;
- source-order tool publication, one-read immutable source validation, and shared tool-batch reconstruction isolated in `runtime/drive/tool-placement.ts`;
- same-call tool status defenses, unchecked `toolOperation()` casts, and their result unions removed; genuine sibling/progress/memo checks retained;
- centralized lane-storage classification without duplicate projection reads;
- checkpoint dead-check removal and exact checkpoint preservation;
- shared transcript mechanics in `runtime/transcript.ts`;
- narrower `LanePatch`, no redundant Drive promise-settlement flag, and no unused operation reject arm.
At simplification completion, the largest files were `runtime/lane.ts` (~996), `runtime/drive/tools.ts` (~663), `runtime/drive/checkpoint.ts` (~452), and `runtime/drive/response.ts` (~417).
### WP05 structural status
M6 established structural execution on the simplified substrate. Its first coherent slice added:
- shared compaction-bounded transcript/context reads and committed-entry event decoration;
- caller-owned one-provider-request seams for compaction and branch summaries while preserving retrying behavior for existing non-harness callers;
- direct procedures for every structural `at` leaf in `runtime/drive/structural.ts`;
- once-per-trigger threshold routing and atomic overflow preparation publication;
- per-request structural intent and usage settlement, attempt retry/recovery, hook decisions, compaction/navigation publication, and terminal cleanup;
- focused coverage for threshold/overflow entry, split-turn request accounting, generated and hook results, retry/cap/recovery, model absence, mid-request durable cancellation, navigation, and preparation corruption.
R1a, R2, and R3 then moved queued input to the lane, replaced hydrated family outcomes with immutable operation records, and collapsed the family cross-product to 13 neutral leaves. R1b replaced transitional checkpoint drains and marker fields with one shared atomic boundary planner, a transcript-derived threshold guard, and off-line finish-hook replanning. M7 removed drained control, added atomic drain-and-return cancellation, reconciled all 13 leaves, and installed the total direct dispatcher.
M8 added primitive/convenience lane surfaces, one ordered tagged inbox, idle ownership, snapshot/event replication, `reduceLaneSnapshot`, and remote resnapshot. M9 reconciled the normative specification. M10 forwards exactly one stable ordinary-provider identity derived from Session metadata id plus lane name; structural summaries keep fresh request identities. At M10 completion the runtime is 7,410 lines, `runtime/drive` is 4,147, `lane.ts` is 1,992, `structural.ts` is 1,221, and the new normative reducer is 193 lines. Relative to the reviewed M7 checkpoint, the public/replication surface added 1,130 runtime lines; the durable 13-leaf procedure model stayed unchanged.
Keep the visible durable procedure order:
`prepare → publish intent → perform effect → publish outcome`.
Do not introduce a generic Procedure interface, runner, scheduler, graph, callback plan, or dependency facade. Historical work-package documents remain unchanged.
## Core model
A Lane is one process-local actor over one durable lane projection.
- `Lane.state` is authoritative for tip, configuration, current operation, control, inbox IDs, and the latest operation ID.
- Every supported mutation commits through the Session mutation line and publishes the matching `Lane.state` before releasing it.
- One lane-owned Drive is the sole writer that advances an operation state.
- Same-operation callers observe the same Drive. No caller owns it.
- Invocation cancellation stops only that caller's observation after Drive installation.
- `requestAbort` is the only durable operation cancellation.
- Close seals mutation admission. It neither mutates operation state nor replaces a Drive.
- Process loss destroys all live continuations; recovery starts from the durable state after attachment.
- Session reads during execution dereference content named by `Lane.state`; they never rediscover control state.
## Durable state
`OperationState` has one discriminator, `at`, with 13 direct leaves:
- `starting`
- `checkpoint`
- `assistant.ready`
- `assistant.effect_pending`
- `assistant.retry_wait`
- `tools`
- `deferred.suspended`
- `deferred.effect_pending`
- `summary.deciding`
- `summary.ready`
- `summary.effect_pending`
- `summary.retry_wait`
- `navigation.ready_to_commit`
Every leaf carries one uniform `OperationScope`. The summary quadruple carries one `SummaryTask`; its closed `ResultBoundary` selects in-run checkpoint resumption, standalone finish, or navigation commit. The summary algorithm is derived from that boundary rather than duplicated in state. `Control` remains orthogonal. `ToolBatch` remains a child state machine because parallel tool children genuinely mutate sibling call statuses concurrently.
## State and content boundary
Procedures never read these addresses to decide execution:
- `laneState`
- `operationMeta`
- `operationState`
- `branchTip`
- `laneConfig`
- `operationResult`
They use the current `Lane.state` supplied by the mutation line.
Storage reads remain only for content and cleanup:
- prompt, assistant, deferred-source, final-assistant, and completed-tool entries;
- compaction-bounded branch context;
- `pendingEntry` payloads;
- assistant-frame lists;
- tool arguments, memos, and checkpoints;
- structural preparations;
- staged tool outcomes;
- operation-owned prefix scans required for terminal cleanup.
## Concurrency model
### Operation advancement
While a Drive procedure is alive, no supported concurrent actor can change or remove its operation state:
- inbox methods change only inbox fields;
- `requestAbort` changes only `control`;
- close prevents later mutation admission;
- another same-operation caller joins the existing Drive;
- another operation cannot be accepted while the lane is busy;
- process crash removes the continuation itself.
Consequently, ordinary procedure transitions do not recheck operation existence, operation ID, operation kind, `at`, attempt identity, or nested status. Those are procedure preconditions established by dispatch, not supported races.
The remaining real races are:
1. cancellation arriving before effect admission or before settlement;
2. inbox arrival before checkpoint routing or terminal finish;
3. parallel tool children staging and materializing sibling outcomes;
4. queued frame/checkpoint/memo writes racing effect settlement;
5. retry timers racing cancellation or close;
6. deferred permit consumption;
7. accept/claim serialization on the Session line.
### Cancellation boundaries
Cancellation is checked in three places only:
1. the drive loop before ordinary dispatch;
2. the gate immediately before an external effect starts;
3. effect settlement, which sees current `control` and commits the appropriate cancelled result.
An ordinary transition helper may centrally decline progress when current control is `cancel_requested`. Procedures do not repeat that branch manually.
### Close
Closing the harness:
1. marks harness/lane admission closed;
2. seals and drains the Session mutation line;
3. rejects client observations through a harness-close observation promise;
4. keeps every detached pass promise observed so a later rejection is never unhandled;
5. closes the Session after admitted mutations drain.
Close performs no durable write, installs no replacement Drive, and creates no ownership-loss state. A late effect may return, but its mutation is rejected with `HarnessClosed`.
Whether close also signals process-local provider/tool work is a resource-cleanup policy, not durable state-machine behavior. Do not couple it to Drive replacement or recovery.
## Small concrete mutation API
Replace repeated `LaneCommand` ceremony in procedures with two concrete Lane operations. They are not a scheduler, graph, or action interpreter.
### `continueOperation`
Used for ordinary non-terminal progress.
- Enters the Session mutation line.
- Receives the current authoritative Lane projection and operation state.
- If control is cancelled, returns explicit `cancel_requested` without invoking the semantic planner; callers cannot confuse cancellation with a planner value.
- The planner supplies procedure-specific writes, the next complete `OperationState`, materialization, and events.
- The helper appends the `operationState` write and publishes the matching process-local operation projection.
- It does not verify an expected state or `at` value.
### `settleOperation`
Used after an admitted provider/tool/structural effect and for genuine parallel child transitions.
- Enters the Session mutation line even when control is cancelled.
- Supplies current control/inbox fields and the process-local effect result to the semantic settlement planner.
- Atomically commits payload, usage, tip movement, cleanup, and the classified next state.
- Appends the canonical operation-state write and publishes the matching projection.
- When the planner returns a terminal decision, appends the immutable `operationResult`, idle `laneState` with `lastOperationId`, and the idle process-local projection.
The caller supplies the typed outcome, cleanup/publication writes, last result, and event. Terminal business decisions remain visible in the owning procedure.
## Durable procedure shape
Effectful procedures expose four concrete phases:
```text
prepare immutable inputs
→ publish durable effect intent
→ perform the external effect
→ publish one durable outcome
```
The intent phase must remain visible and precede the external effect. Otherwise a crash after the effect but before its intent would leave no recoverable unknown-outcome marker. Each procedure uses concrete functions such as `prepareGeneration`, `publishGenerationIntent`, `performGeneration`, and `publishResponse`; there is no generic Procedure abstraction.
## Drive lifecycle
Delete the installer-owned model.
- Remove `installerSignal`.
- Remove `DriveAbandoned`.
- Remove `LostOwnership` and `lost_ownership` from procedure results.
- Remove exact-object ABA fencing and `commandDriveOwned`.
- Remove `finalizedOutcome` and planned external-finalization owner retention unless the Flue investigation establishes a concrete requirement.
- Keep `activeDrive` only as the lane's install/join slot.
- Installation transfers work to the lane. Every caller, including the installer, observes completion with its own invocation Context.
- A caller signal abort before installation installs nothing. After installation it only rejects that caller's observation.
- The Drive is removed after its pass settles or reaches a durable wait. No live pass is replaced in-process.
`requestAbort` retains the two-part gate ordering:
```text
beginAbort before cancellation mutation
commit cancel_requested
signalAbort after commit
```
This prevents a new effect from entering while the durable marker is being committed.
## Checks that remain
Do not remove validation of external or referenced content:
- required entry existence and role;
- pending payload kind;
- deferred handle identity;
- provider stream protocol ordering;
- response stop-reason invariants;
- UUIDv7 follower timestamp parsing;
- configured model/tool availability;
- tool-call source index and staged result identity;
- parallel tool call status and ready-prefix placement;
- progress/memo invocation identity;
- retry timestamp and deferred permit arithmetic.
These validate data or genuine child concurrency. They are not defensive revalidation of the operation state machine.
## Completed staged implementation plan
### Stage 1 — Canonical flat durable types
Files:
- `src/harness/session/types.ts`
- compile-only consumers in `src/harness/runtime/`, restore, conformance helpers, and focused tests
- `docs/harness.md`
- `docs/work-packages/05-direct-durable-drive.md`
Actions:
- Replace nested operation state declarations with the flat `at` union.
- Preserve every durable datum without compatibility aliases.
- Update pattern matching mechanically without changing behavior.
- Keep ToolBatch/ToolCall nested.
- Update normative documentation in the same change.
Exit: `npm run check`; existing focused runtime tests pass; no `phase.kind`, generation `status`, deferred `status`, or structural decision `status` remains in canonical operation state.
### Stage 2 — Add canonical transition operations
Files:
- `src/harness/runtime/lane.ts`
- `src/harness/runtime/types.ts`
- focused Lane tests
Actions:
- Add `continueOperation` and `settleOperation`, including the terminal-decision suffix.
- Pair every durable operation-state write with process-local projection publication in one implementation.
- Centralize ordinary cancellation diversion.
- Trust the dispatcher-established current leaf; perform no expected-state check.
- Keep procedure-specific writes and event builders visible at call sites.
Exit: focused tests prove durable state and `Lane.state` remain identical after every helper commit.
### Stage 3 — Convert starting, checkpoint, and assistant
Files:
- `runtime/drive/checkpoint.ts`
- `runtime/drive/generation.ts`
- `runtime/drive/recovery.ts`
- `runtime/progress.ts`
Actions:
- Remove repeated operation/null/kind/state checks and `same*` predicates.
- Convert ordinary progress to `continueOperation`.
- Convert assistant settlement to `settleOperation`.
- Keep only checkpoint inbox/finish races and progress-channel ownership checks.
- Fold assistant recovery into the effect-pending handler if that is smaller.
Exit: no control-state storage reads; no repeated assistant state verification; focused generation tests pass.
### Stage 4 — Convert deferred and tools
Files:
- `runtime/drive/deferred.ts`
- `runtime/drive/tools.ts`
- `runtime/progress.ts`
Actions:
- Share assistant/deferred response-entry, usage, and tool-plan construction where their semantics match.
- Preserve deferred permit and handle checks.
- Remove top-level operation-state revalidation.
- Preserve per-call tool status merging, completion-order staging, source-order placement, memo fencing, and progress fencing.
- Use one tool-batch procedure for live, recovery, and cancellation modes instead of separate ownership-result paths.
Exit: tool status checks exist only for genuine sibling concurrency; focused deferred/tool/progress tests pass.
### Stage 5 — Remove ownership-loss machinery
Files:
- `runtime/types.ts`
- `runtime/lane.ts`
- `execution/effect-gate.ts`
- all existing `runtime/drive/*.ts`
- associated focused tests
Actions:
- Remove `LostOwnership`, `commandDriveOwned`, exact Drive checks, `installerSignal`, `DriveAbandoned`, and `finalizedOutcome`.
- Make all drive callers observation peers.
- Add observation-only Context cancellation to Drive completion waiting.
- Keep `activeDrive` only for install/join arbitration.
- Remove ABA/replacement tests and replace them with install/join/observation-cancellation tests.
Exit: grep finds no ownership-loss or installer-abandonment vocabulary in production runtime.
### Stage 6 — Close and fault
Files:
- `runtime/harness.ts`
- `runtime/lane.ts`
- Drive observation helper
- lifecycle tests
Actions:
- Seal mutation admission and drain admitted mutations.
- Reject client observations on close/fault without replacing a Drive or changing durable operation state.
- Observe detached pass failures.
- Verify a late effect cannot commit after close.
- Decide separately whether to signal local effects for resource cleanup.
Exit: close and process loss leave the same durable restart point; no close path writes cancellation or synthetic settlement.
### Stage 7 — Finish WP05 on the simpler substrate
Files:
- `runtime/drive/structural.ts`
- `runtime/drive/reconcile.ts`
- `runtime/drive.ts`
- `runtime/lane.ts` public surfaces
Actions:
- Implement structural generation directly over flat leaves and the transition operations.
- Implement cancellation reconciliation as one flat-state switch.
- Implement the total drive switch as one `state.at` switch.
- Wire public claim/join/observation and convenience methods only after every leaf is total.
External finalization is excluded unless the Flue investigation identifies a concrete, current caller that cannot be expressed through close, explicit abort, recovery, or offline administration.
## Validation
After every code stage:
```bash
npm run check
```
Run each modified focused test file from the package root. Do not run the full Vitest suite directly. Public drive was enabled only after the final stage made every leaf total.
Final audit:
```bash
rg 'lost_ownership|LostOwnership|DriveAbandoned|commandDriveOwned|installerSignal|finalizedOutcome' packages/agent/src/harness
rg 'operationState\(|laneState\(|branchTip\(|laneConfig\(' packages/agent/src/harness/runtime/drive
```
The second audit may match write constructors, but no reader call may use those control addresses.

383
third_party/pi/packages/agent/docs/telemetry-schema.md vendored Normal file
View File

@@ -0,0 +1,383 @@
# Pi Agent Telemetry Schemas
<!-- Generated by generate-telemetry-docs.ts. Do not edit manually. -->
## AI request schema
Schema version: 1
### `pi.ai.request`
One logical request to an AI provider
- Parents: root or any caller span
- Default status: `ok`
- Error when: The operation throws or returns an error result
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.ai.operation` | `string` | yes | stream, fetch_deferred, cancel_deferred, generate_images | | Logical provider operation |
| `pi.ai.provider` | `string` | yes | | | Selected provider id |
| `pi.ai.model` | `string` | yes | | | Requested model id |
| `pi.ai.api` | `string` | yes | | | Provider API id |
| `pi.ai.streaming` | `boolean` | yes | | | Whether this operation returns a stream |
| `pi.ai.deferred` | `boolean` | no | | | Whether the operation requests or participates in deferred execution |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.ai.response.model` | `string` | | | Concrete response model |
| `pi.ai.response.id` | `string` | | high cardinality | Provider response id |
| `pi.ai.response.stop_reason` | `string` | stop, length, tool_use, error, aborted, deferred | | Normalized terminal response reason |
| `pi.ai.http.status_code` | `number` | | | Final HTTP status |
| `pi.ai.usage.input_tokens` | `number` | | | Reported input tokens |
| `pi.ai.usage.output_tokens` | `number` | | | Reported output tokens |
| `pi.ai.usage.cache_read_tokens` | `number` | | | Reported cache-read tokens |
| `pi.ai.usage.cache_write_tokens` | `number` | | | Reported cache-write tokens |
| `pi.ai.usage.reasoning_tokens` | `number` | | | Reported reasoning tokens |
| `pi.ai.usage.total_tokens` | `number` | | | Reported total tokens |
| `pi.ai.usage.cost` | `number` | | | Reported total cost |
| `pi.ai.stream.chunk_count` | `number` | | | Streamed update chunk count |
| `pi.ai.stream.time_to_first_chunk_ms` | `number` | | | Elapsed milliseconds to first update chunk |
| `pi.ai.error.type` | `string` | | low cardinality | Provider or transport error class |
#### Events
No declared span events.
## Harness schema
Schema version: 1
### `pi.harness.run`
One admitted in-process run invocation
- Parents: root or caller-owned external span
- Default status: `ok`
- Error when: The run fails or throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.session.id` | `string` | yes | | high cardinality | Session id |
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.operation.recovery` | `boolean` | yes | | | Whether this invocation resumes durable work |
| `pi.operation.kind` | `string` | yes | run | | Run operation kind |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.operation.outcome` | `string` | completed, aborted, failed, suspended | | Run invocation outcome |
| `pi.error.code` | `string` | | low cardinality | Stable operation error code |
| `pi.error.type` | `string` | | low cardinality | Low-cardinality operation error class |
#### Events
No declared span events.
### `pi.harness.compaction`
One admitted in-process manual compaction invocation
- Parents: root or caller-owned external span
- Default status: `ok`
- Error when: The compaction fails or throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.session.id` | `string` | yes | | high cardinality | Session id |
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.operation.recovery` | `boolean` | yes | | | Whether this invocation resumes durable work |
| `pi.operation.kind` | `string` | yes | compaction | | Compaction operation kind |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.operation.outcome` | `string` | completed, declined, aborted, failed | | Compaction invocation outcome |
| `pi.error.code` | `string` | | low cardinality | Stable operation error code |
| `pi.error.type` | `string` | | low cardinality | Low-cardinality operation error class |
#### Events
No declared span events.
### `pi.harness.navigation`
One admitted in-process navigation invocation
- Parents: root or caller-owned external span
- Default status: `ok`
- Error when: The navigation fails or throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.session.id` | `string` | yes | | high cardinality | Session id |
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.operation.recovery` | `boolean` | yes | | | Whether this invocation resumes durable work |
| `pi.operation.kind` | `string` | yes | navigation | | Navigation operation kind |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.operation.outcome` | `string` | completed, declined, aborted, failed | | Navigation invocation outcome |
| `pi.error.code` | `string` | | low cardinality | Stable operation error code |
| `pi.error.type` | `string` | | low cardinality | Low-cardinality operation error class |
#### Events
No declared span events.
### `pi.harness.checkpoint`
One run checkpoint
- Parents: `pi.harness.run`
- Default status: `ok`
- Error when: Checkpoint work throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.checkpoint.kind` | `string` | yes | normal, abort_reconcile | | Checkpoint purpose |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| _none_ | | | | |
#### Events
No declared span events.
### `pi.harness.turn`
One assistant response and its tool batch
- Parents: `pi.harness.run`
- Default status: `ok`
- Error when: Turn work throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.turn.id` | `string` | yes | | high cardinality | Invocation-local turn id |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| _none_ | | | | |
#### Events
No declared span events.
### `pi.harness.step`
One durable retry attempt
- Parents: `pi.harness.turn`, `pi.harness.checkpoint`, `pi.harness.compaction`, `pi.harness.navigation`
- Default status: `ok`
- Error when: The attempt retries, fails, or throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.step.kind` | `string` | yes | assistant, compaction, branch_summary | | Retryable step kind |
| `pi.step.attempt` | `number` | yes | | | One-based durable attempt number |
| `pi.compaction.reason` | `string` | no | manual, threshold, overflow | | Compaction trigger |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.step.outcome` | `string` | succeeded, retry, failed, aborted, deferred, overflow | | Attempt outcome |
#### Events
No declared span events.
### `pi.harness.tool`
One raw phase-2 tool execution
- Parents: `pi.harness.turn`, `pi.harness.run`
- Default status: `ok`
- Error when: Raw phase-2 execution returns an error
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.turn.id` | `string` | no | | high cardinality | Invocation-local live turn id |
| `pi.tool.name` | `string` | yes | | | Tool name |
| `pi.tool.call_id` | `string` | yes | | high cardinality | Tool call id |
| `pi.tool.replay` | `string` | yes | never, safe | | Declared replay policy |
| `pi.tool.recovery` | `boolean` | yes | | | Whether this is recovery execution |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.tool.is_error` | `boolean` | | | Whether raw phase-2 execution returned an error |
#### Events
No declared span events.
### `pi.harness.hook`
One registered hook handler invocation
- Parents: root or any caller span
- Default status: `ok`
- Error when: The handler throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | no | | high cardinality | Durable operation id when accepted |
| `pi.hook.name` | `string` | yes | before_run, before_drive, before_run_end, transform_context, before_request, before_payload, after_response, before_tool, after_tool, before_compaction, before_navigation | | Hook name |
| `pi.hook.registration_id` | `string` | no | | | Optional hook registration metadata |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.hook.outcome` | `string` | completed, skipped, blocked, failed | | Handler outcome |
#### Events
No declared span events.
### `pi.harness.sleep`
One retry delay
- Parents: `pi.harness.run`, `pi.harness.compaction`, `pi.harness.navigation`, `pi.harness.turn`, `pi.harness.checkpoint`
- Default status: `ok`
- Error when: Sleep work throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.sleep.delay_ms` | `number` | yes | | | Requested delay in milliseconds |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.sleep.outcome` | `string` | elapsed, aborted | | Delay outcome |
#### Events
No declared span events.
### `pi.harness.event_handler`
One passive event listener invocation
- Parents: root or any caller span
- Default status: `ok`
- Error when: The listener throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.event.type` | `string` | yes | run_start, run_resume, run_suspend, operation_abort, run_end, fault, handler_error, turn_start, turn_end, retry_scheduled, retry_start, retry_end, message_start, message_update, message_end, tool_start, tool_update, tool_end, entry_added, queue_update, value_update, config_update, compaction_start, compaction_end, navigation_start, navigation_end, lane_created, usage | low cardinality | Delivered harness event type |
| `pi.lane.name` | `string` | no | | high cardinality | Lane name for lane-scoped events |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| _none_ | | | | |
#### Events
No declared span events.
### `pi.session.write`
One committed session transaction
- Parents: root or any caller span
- Default status: `ok`
- Error when: Storage rejects the transaction
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.session.id` | `string` | yes | | high cardinality | Session id |
| `pi.lane.name` | `string` | no | | high cardinality | Lane name when supplied by the caller |
| `pi.operation.id` | `string` | no | | high cardinality | Durable operation id when supplied by the caller |
| `pi.session.item_count` | `number` | yes | | | Number of writes in the transaction |
| `pi.session.item_kinds` | `string[]` | yes | elements: entry, usage, value, list | | Distinct write kinds in the transaction |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.session.first_seq` | `number` | | | First committed sequence in the transaction |
| `pi.session.last_seq` | `number` | | | Last committed sequence in the transaction |
#### Events
No declared span events.

Some files were not shown because too many files have changed in this diff Show More