alexion/dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(vendor): import pi 1.3.0

Browse Source 
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree.

git-subtree-dir: third_party/pi

git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
This commit is contained in:
alexion
2026-10-15 17:58:48 -04:00
parent 719370c8f7
commit 6081e3e325
1743 changed files with 205460 additions and 131705 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
third_party/pi/.gitattributes vendored
View File

@@ -9,6 +9,9 @@
# Shell scripts should keep LF
*.sh text eol=lf
# Archived patch artifacts preserve whitespace exactly
packages/agent/docs/**/*.diff -whitespace
# Common binary assets
*.png binary
*.jpg binary

16
third_party/pi/.github/APPROVED_CONTRIBUTORS vendored
View File

@@ -389,3 +389,19 @@ y-nk pr
edverma pr
Charlie0113-T pr
RedBeard0531 pr
metaist pr
jsanter27 pr
philfreo pr
adawalli pr
btkostner pr
Jay-Madden issue
lalyos pr

42
third_party/pi/.github/workflows/build-binaries.yml vendored
View File

@@ -142,7 +142,14 @@ jobs:
actions: read
env:
RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }}
SOURCE_REF: ${{ github.event.inputs.source_ref || github.event.inputs.tag || github.ref_name }}
steps:
- name: Checkout smoke tests
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ env.SOURCE_REF }}
persist-credentials: false
- name: Download binary archives
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
@@ -198,6 +205,7 @@ jobs:
run: |
"${{ steps.binary.outputs.binary }}" --help
"${{ steps.binary.outputs.binary }}" --version
node scripts/smoke-test-codemode-binary.mjs "${{ steps.binary.outputs.binary }}"
stage-github-release:
runs-on: ubuntu-latest
@@ -293,9 +301,17 @@ jobs:
exit 1
fi
# The pi-env daemon for every remote system @earendil-works/pi-env supports; the npm package ships them all.
env-daemons:
uses: ./.github/workflows/env-daemons.yml
permissions:
contents: read
with:
ref: ${{ github.event.inputs.source_ref || github.event.inputs.tag || github.ref_name }}
publish-npm:
runs-on: ubuntu-latest
needs: stage-github-release
needs: [stage-github-release, env-daemons]
environment: npm-publish
permissions:
contents: read
@@ -326,12 +342,33 @@ jobs:
- name: Install dependencies
run: npm ci --ignore-scripts
- name: Download pi-env daemons
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
pattern: pi-env-*
path: packages/env/bin
- name: Check pi-env daemons
run: |
set -euo pipefail
for name in linux-x64 linux-arm64 android-arm64 android-x64 darwin-arm64 darwin-x64; do
test -s "packages/env/bin/pi-env-${name}/pi-env"
done
for name in windows-x64 windows-arm64; do
test -s "packages/env/bin/pi-env-${name}/pi-env.exe"
done
chmod +x packages/env/bin/*/pi-env*
ls -lR packages/env/bin
- name: Build
run: npm run build
- name: Check
run: npm run check
- name: Build the pi-env daemon for tests
run: npm run build:daemon --workspace packages/env
- name: Test
run: npm test
@@ -343,6 +380,9 @@ jobs:
npm install -g npm@11.16.0 --ignore-scripts
npm --version
- name: Validate npm tarball publication with provenance
run: node scripts/publish.mjs --dry-run
- name: Publish npm packages
run: node scripts/publish.mjs

23
third_party/pi/.github/workflows/ci.yml vendored
View File

@@ -38,5 +38,28 @@ jobs:
- name: Check
run: npm run check
- name: Build the pi-env daemon
run: npm run build:daemon --workspace packages/env
- name: Test
run: npm test
mcp-conformance:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: npm
- name: Install dependencies
run: npm ci --ignore-scripts
# Runs pi's MCP client from source against the pinned official conformance suite and fails on
# regressions against packages/coding-agent/test/mcp-conformance/baseline.json.
- name: MCP client conformance
run: npm run test:mcp-conformance

89
third_party/pi/.github/workflows/env-daemons.yml vendored Normal file
View File

@@ -0,0 +1,89 @@
name: pi-env daemons
# Builds the pi-env daemon for every remote system @earendil-works/pi-env supports, on native runners, as artifacts
# named pi-env-<platform>-<arch>. Build Binaries calls this before publishing npm packages; the Environments workflow
# calls it to test the release builds on every change.
on:
workflow_call:
inputs:
ref:
description: Source ref to build
required: false
type: string
workflow_dispatch:
permissions:
contents: read
jobs:
build:
strategy:
fail-fast: true
matrix:
include:
- { runner: ubuntu-latest, target: x86_64-unknown-linux-musl, name: linux-x64 }
- { runner: ubuntu-24.04-arm, target: aarch64-unknown-linux-musl, name: linux-arm64 }
- { runner: ubuntu-latest, target: aarch64-linux-android, name: android-arm64, android: aarch64-linux-android24 }
- { runner: ubuntu-latest, target: x86_64-linux-android, name: android-x64, android: x86_64-linux-android24 }
- { runner: macos-latest, target: aarch64-apple-darwin, name: darwin-arm64 }
- { runner: macos-latest, target: x86_64-apple-darwin, name: darwin-x64 }
- { runner: windows-latest, target: x86_64-pc-windows-msvc, name: windows-x64 }
- { runner: windows-11-arm, target: aarch64-pc-windows-msvc, name: windows-arm64 }
runs-on: ${{ matrix.runner }}
defaults:
run:
shell: bash
working-directory: packages/env/daemon
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ inputs.ref || github.ref }}
persist-credentials: false
- name: Add target
run: rustup target add ${{ matrix.target }}
- name: Configure the Android NDK linker
if: matrix.android
# A fixed NDK, installed when the runner image does not have it.
env:
NDK_VERSION: 27.2.12479018
run: |
ndk="$ANDROID_HOME/ndk/$NDK_VERSION"
if [ ! -d "$ndk" ]; then
# `yes` ends with SIGPIPE once the licenses are accepted.
{ yes || true; } | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --install "ndk;$NDK_VERSION" > /dev/null
fi
linker="$ndk/toolchains/llvm/prebuilt/linux-x86_64/bin/${{ matrix.android }}-clang"
test -x "$linker"
variable="CARGO_TARGET_$(echo '${{ matrix.target }}' | tr 'a-z-' 'A-Z_')_LINKER"
echo "$variable=$linker" >> "$GITHUB_ENV"
- name: Build
# Static C runtime on Windows, so the daemon needs no Visual C++ redistributable on the remote machine.
env:
RUSTFLAGS: ${{ contains(matrix.target, 'windows') && '-C target-feature=+crt-static' || '' }}
run: cargo build --release --locked --target ${{ matrix.target }}
- name: Collect
run: |
suffix=""
case "${{ matrix.target }}" in *windows*) suffix=".exe" ;; esac
mkdir -p "$GITHUB_WORKSPACE/pi-env-${{ matrix.name }}"
cp "target/${{ matrix.target }}/release/pi-env$suffix" "$GITHUB_WORKSPACE/pi-env-${{ matrix.name }}/pi-env$suffix"
ls -l "$GITHUB_WORKSPACE/pi-env-${{ matrix.name }}"
- name: Run the native build
if: ${{ !matrix.android && !(matrix.target == 'x86_64-apple-darwin') }}
run: |
suffix=""
case "${{ matrix.target }}" in *windows*) suffix=".exe" ;; esac
"$GITHUB_WORKSPACE/pi-env-${{ matrix.name }}/pi-env$suffix" --version
- name: Upload
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: pi-env-${{ matrix.name }}
path: pi-env-${{ matrix.name }}
if-no-files-found: error

200
third_party/pi/.github/workflows/env.yml vendored Normal file
View File

@@ -0,0 +1,200 @@
name: Environments
# Execution environments on every OS: Durable's NodeExecutionEnv and pi-env's daemon and RemoteExecutionEnv, which
# must give the same results. The main CI runs on Linux only. The release builds of the daemon are tested as shipped,
# and so is the npm package's file list.
on:
push:
branches: [main, "durable-env-*"]
paths: ["packages/durable/src/env/**", "packages/durable/src/tools/**", "packages/durable/test/env-*", "packages/env/**", ".github/workflows/env.yml", ".github/workflows/env-daemons.yml"]
pull_request:
branches: [main]
paths: ["packages/durable/src/env/**", "packages/durable/src/tools/**", "packages/durable/test/env-*", "packages/env/**", ".github/workflows/env.yml", ".github/workflows/env-daemons.yml"]
workflow_dispatch:
concurrency:
group: env-${{ github.ref }}
cancel-in-progress: true
jobs:
daemons:
uses: ./.github/workflows/env-daemons.yml
permissions:
contents: read
check:
needs: daemons
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: npm
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev fd-find ripgrep
sudo ln -s $(which fdfind) /usr/local/bin/fd
- name: Install dependencies
run: npm ci --ignore-scripts
- name: Build
run: npm run build
- name: Check
run: npm run check
- name: Download the pi-env daemons
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
pattern: pi-env-*
path: packages/env/bin
- name: Check the pi-env package contents
working-directory: packages/env
run: |
npm pack --dry-run --json > "$RUNNER_TEMP/pack.json"
node -e '
const files = new Set(JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))[0].files.map((file) => file.path));
const expected = ["dist/index.js", "dist/index.d.ts", "dist/ssh.js", "README.md", "CHANGELOG.md", "docs/protocol.md"];
for (const name of ["linux-x64", "linux-arm64", "android-arm64", "android-x64", "darwin-arm64", "darwin-x64"]) expected.push(`bin/pi-env-${name}/pi-env`);
for (const name of ["windows-x64", "windows-arm64"]) expected.push(`bin/pi-env-${name}/pi-env.exe`);
const missing = expected.filter((file) => !files.has(file));
const unexpected = [...files].filter((file) => file.startsWith("src/") || file.startsWith("daemon/") || file.startsWith("test/"));
if (missing.length || unexpected.length) { console.error({ missing, unexpected }); process.exit(1); }
console.log(`${files.size} files`);
' "$RUNNER_TEMP/pack.json"
# The pi-env tests against each release build of the daemon, as the npm package ships it.
release-daemons:
needs: daemons
strategy:
fail-fast: false
matrix:
include:
- { os: ubuntu-latest, name: linux-x64 }
- { os: ubuntu-24.04-arm, name: linux-arm64 }
- { os: macos-latest, name: darwin-arm64 }
# Under Rosetta on the Apple silicon runner.
- { os: macos-latest, name: darwin-x64 }
- { os: windows-latest, name: windows-x64 }
- { os: windows-11-arm, name: windows-arm64 }
runs-on: ${{ matrix.os }}
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: npm
- name: Install dependencies
run: npm ci --ignore-scripts
- name: Install an SSH server for the bootstrap tests
if: runner.os == 'Linux'
run: command -v sshd || test -x /usr/sbin/sshd || (sudo apt-get update && sudo apt-get install -y openssh-server)
- name: Install Rosetta
if: matrix.name == 'darwin-x64'
run: softwareupdate --install-rosetta --agree-to-license
- name: Download the daemon
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: pi-env-${{ matrix.name }}
path: ${{ runner.temp }}/daemon
- name: pi-env tests
working-directory: packages/env
run: |
daemon="$RUNNER_TEMP/daemon/pi-env"
case "${{ matrix.name }}" in windows-*) daemon="$daemon.exe" ;; esac
chmod +x "$daemon"
file "$daemon" || true
PI_ENV_DAEMON="$daemon" npx vitest --run
tests:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest, windows-11-arm]
runs-on: ${{ matrix.os }}
defaults:
run:
shell: bash
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: npm
- name: Install dependencies
run: npm ci --ignore-scripts
- name: Install an SSH server for the bootstrap tests
if: runner.os == 'Linux'
run: command -v sshd || test -x /usr/sbin/sshd || (sudo apt-get update && sudo apt-get install -y openssh-server)
- name: Daemon format, lint and unit tests
working-directory: packages/env/daemon
run: |
cargo fmt --check
cargo clippy --locked --all-targets -- -D warnings
cargo test --locked
- name: Build the daemon
working-directory: packages/env
run: npm run build:daemon
- name: Durable env tests
working-directory: packages/durable
run: npx vitest --run test/env-node.test.ts test/env-node-spill.test.ts test/env-node-conformance.test.ts test/env-line-scan.test.ts test/tools.test.ts test/tools-read-differential.test.ts
- name: pi-env tests
working-directory: packages/env
run: npx vitest --run
- name: Start Windows' OpenSSH server
if: matrix.os == 'windows-latest'
shell: pwsh
run: |
if (-not (Get-Service sshd -ErrorAction SilentlyContinue)) {
Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 | Out-Null
}
Start-Service sshd
$key = Join-Path $env:RUNNER_TEMP 'ssh_client_key'
& "$env:WINDIR\System32\OpenSSH\ssh-keygen.exe" -q -t ed25519 -N '' -f $key
icacls $key /inheritance:r /grant "${env:USERNAME}:F" | Out-Null
# The runner's user is an administrator; sshd reads administrators' keys from ProgramData.
$authorized = "$env:ProgramData\ssh\administrators_authorized_keys"
Get-Content "$key.pub" | Set-Content -Path $authorized -Encoding ascii
icacls $authorized /inheritance:r /grant "Administrators:F" /grant "SYSTEM:F" | Out-Null
- name: pi-env tests over Windows' OpenSSH, with cmd.exe and then PowerShell as its default shell
if: matrix.os == 'windows-latest'
working-directory: packages/env
env:
PI_ENV_SSH_HOST: 127.0.0.1
PI_ENV_SSH_KEY: ${{ runner.temp }}\ssh_client_key
run: |
export PI_ENV_SSH_USER="$USERNAME"
export PI_ENV_SSH_PROGRAM="$WINDIR\System32\OpenSSH\ssh.exe"
npx vitest --run test/ssh-external.test.ts
powershell -NoProfile -Command "New-ItemProperty -Path 'HKLM:\SOFTWARE\OpenSSH' -Name DefaultShell -Value 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe' -PropertyType String -Force | Out-Null"
PI_ENV_SSH_SHELL=powershell npx vitest --run test/ssh-external.test.ts

40
third_party/pi/.github/workflows/issue-triage-labels.yml vendored
View File

@@ -19,6 +19,12 @@ jobs:
const LAST_READ_LABEL = 'last-read';
const TO_DISCUSS_LABEL = 'to-discuss';
const INPROGRESS_LABEL = 'inprogress';
// Applying one of these labels assigns the user and reopens the issue if it is closed.
const LABEL_ASSIGNEES = new Map([
['mcp', 'mitsuhiko'],
['codemode', 'mitsuhiko'],
['durable', 'badlogic'],
]);
function issueHasLabel(issue, labelName) {
return (issue.labels ?? []).some((label) => label.name === labelName);
@@ -47,9 +53,13 @@ jobs:
}
}
async function removeClosedStateLabels(issueNumber, issue) {
await removeLabelIfPresent(issueNumber, issue, UNTRIAGED_LABEL);
await removeLabelIfPresent(issueNumber, issue, NO_ACTION_LABEL);
}
if (context.payload.action === 'reopened') {
await removeLabelIfPresent(context.issue.number, context.payload.issue, UNTRIAGED_LABEL);
await removeLabelIfPresent(context.issue.number, context.payload.issue, NO_ACTION_LABEL);
await removeClosedStateLabels(context.issue.number, context.payload.issue);
return;
}
@@ -58,6 +68,32 @@ jobs:
return;
}
const labelAssignee =
context.payload.action === 'labeled' ? LABEL_ASSIGNEES.get(context.payload.label?.name) : undefined;
if (labelAssignee) {
const issueNumber = context.issue.number;
await github.rest.issues.addAssignees({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issueNumber,
assignees: [labelAssignee],
});
console.log(`Assigned ${labelAssignee} to #${issueNumber}`);
if (context.payload.issue.state === 'closed') {
await github.rest.issues.update({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issueNumber,
state: 'open',
});
console.log(`Reopened #${issueNumber}`);
// Reopening with the workflow token does not trigger the reopened handler above.
await removeClosedStateLabels(issueNumber, context.payload.issue);
}
return;
}
if (context.payload.action !== 'labeled' || context.payload.label?.name !== LAST_READ_LABEL) {
console.log('Not a last-read label event');
return;

189
third_party/pi/.github/workflows/nix.yml vendored Normal file
View File

@@ -0,0 +1,189 @@
name: Nix
# The flake builds with the model catalog revision pinned in
# nix/model-catalog.json. This workflow keeps that pin usable independently of
# releases: after pi.dev publishes a catalog from main, it replaces the pin with
# the newest catalog only if the current one no longer hydrates main (for
# example after a provider was added) or its provider/type/API model groups
# differ from the newest catalog for one of main's providers (for example after
# a provider gained classifier models or switched APIs), builds the flake, and
# commits the new pin to main.
#
# Release tags are built exactly as tagged, with their committed pin. Once all
# systems build, the stable branch is fast-forwarded to the tag so that
# `nix run github:earendil-works/pi/stable` runs the latest release.
on:
push:
tags:
- 'v*.*.*'
pull_request:
branches: [main]
paths:
- '.github/workflows/nix.yml'
- 'flake.lock'
- 'flake.nix'
- 'nix/**'
- 'package-lock.json'
- 'packages/*/package.json'
- 'packages/coding-agent/install-lock/**'
- 'packages/ai/scripts/hydrate-model-catalog.ts'
- 'scripts/update-model-catalog-pin.mjs'
workflow_run:
workflows:
- Publish Model Catalog
types:
- completed
# Catches packaging drift from changes pushed to main without a pull request.
schedule:
- cron: '43 5 * * *'
workflow_dispatch:
permissions: {}
concurrency:
group: nix-${{ (github.event_name == 'pull_request' || github.ref_type == 'tag') && github.ref || 'main' }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
pin:
if: ${{ github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main') }}
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
revision: ${{ steps.pin.outputs.revision }}
updated: ${{ steps.pin.outputs.updated }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
- name: Refresh the model catalog pin if stale
id: pin
run: |
set -euo pipefail
# A release tag must build with its committed pin; it cannot be changed.
if [[ "${GITHUB_REF_TYPE}" != "tag" ]]; then
node scripts/update-model-catalog-pin.mjs --if-stale
fi
revision="$(node -p 'require("./nix/model-catalog.json").revision')"
echo "revision=${revision}" >> "${GITHUB_OUTPUT}"
if git diff --quiet -- nix/model-catalog.json; then
echo "updated=false" >> "${GITHUB_OUTPUT}"
else
echo "updated=true" >> "${GITHUB_OUTPUT}"
if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
echo "::warning::The model catalog pin on main is stale; this build uses ${revision}. The pin is refreshed on main after pi.dev publishes a catalog."
fi
fi
build:
needs: pin
# After a catalog publication, only build when the pin changed.
if: ${{ github.event_name != 'workflow_run' || needs.pin.outputs.updated == 'true' }}
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-latest
system: x86_64-linux
- runner: ubuntu-24.04-arm
system: aarch64-linux
- runner: macos-latest
system: aarch64-darwin
runs-on: ${{ matrix.runner }}
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Use the resolved model catalog pin
env:
REVISION: ${{ needs.pin.outputs.revision }}
run: |
printf '{\n "revision": "%s"\n}\n' "${REVISION}" > nix/model-catalog.json
- name: Install Nix
uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1
with:
github_access_token: ${{ github.token }}
- name: Evaluate all systems
if: ${{ matrix.system == 'x86_64-linux' }}
run: nix flake check --all-systems --no-build
- name: Build
run: nix build -L .
- name: Smoke-test
run: |
./result/bin/pi --version
./result/bin/pi --help > /dev/null
if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then
test "$(./result/bin/pi --version)" = "${GITHUB_REF_NAME#v}"
fi
update-stable:
needs: build
if: ${{ github.ref_type == 'tag' }}
runs-on: ubuntu-latest
permissions:
contents: write
steps:
# Full history, so git can verify that the push to stable is a fast-forward.
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
# Without --force, the push fails unless it is a fast-forward, so stable
# never moves back to an older release.
- name: Fast-forward stable to the release
run: git push origin "${GITHUB_SHA}:refs/heads/stable"
commit-pin:
needs:
- pin
- build
if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && needs.pin.outputs.updated == 'true' }}
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: main
- name: Commit and push the model catalog pin
env:
REVISION: ${{ needs.pin.outputs.revision }}
run: |
set -euo pipefail
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
printf '{\n "revision": "%s"\n}\n' "${REVISION}" > nix/model-catalog.json
git add nix/model-catalog.json
if git diff --staged --quiet; then
echo "Model catalog pin is already ${REVISION}."
exit 0
fi
git commit -m "chore: update Nix model catalog pin"
for attempt in 1 2 3; do
if git push origin HEAD:main; then
exit 0
fi
sleep $((attempt * 5))
git pull --rebase origin main
done
exit 1

2
third_party/pi/.github/workflows/npm-audit.yml vendored
View File

@@ -25,7 +25,7 @@ jobs:
run: npm ci --ignore-scripts --no-audit --no-fund
- name: Audit production vulnerabilities
run: npm audit --omit=dev --audit-level=moderate
run: node scripts/npm-audit.mjs
- name: Verify registry signatures
run: npm audit signatures --omit=dev

1
third_party/pi/.github/workflows/publish-model-catalog.yml vendored
View File

@@ -12,6 +12,7 @@ on:
- '.gitignore'
- 'package.json'
- 'packages/ai/**'
- 'scripts/model-catalog-protocol.ts'
- 'scripts/publish-model-catalog.mjs'
# GitHub schedules use UTC. Run hourly candidates across the CET/CEST
# boundaries; the publish job only uploads at 10:17, 12:17, and 14:17

4
third_party/pi/.gitignore vendored
View File

@@ -1,4 +1,4 @@
node_modules/
node_modules
dist/
.artifacts/
*.log
@@ -37,6 +37,8 @@ out.jsonl
pi-*.html
out.html
packages/coding-agent/binaries/
packages/env/bin/
packages/env/daemon/target/
todo.md
plans/
.pi/hf-sessions/

18
third_party/pi/.husky/pre-commit vendored
View File

@@ -16,6 +16,24 @@ if [ $? -ne 0 ]; then
exit 1
fi
# p2 has its own check (own biome.json, TypeScript 5.9; apps/ is outside the root Biome and tsc configs).
# apps/ exists only in pi-private.
if [ -d apps/p2 ]; then
for file in $STAGED_FILES; do
case "$file" in
apps/p2/*|packages/*|package.json|package-lock.json|tsconfig.json|tsconfig.base.json)
echo "Checking apps/p2..."
npm --prefix apps/p2 run check
if [ $? -ne 0 ]; then
echo "❌ apps/p2 checks failed."
exit 1
fi
break
;;
esac
done
fi
RUN_BROWSER_SMOKE=0
for file in $STAGED_FILES; do
case "$file" in

4
third_party/pi/.pi/prompts/cl.md vendored
View File

@@ -50,5 +50,5 @@ Sections (in order):
- `### Removed` - Removed features
Attribution:
- Internal: `Fixed foo ([#123](https://github.com/earendil-works/pi-mono/issues/123))`
- External: `Added bar ([#456](https://github.com/earendil-works/pi-mono/pull/456) by [@user](https://github.com/user))`
- Internal: `Fixed foo ([#123](https://github.com/earendil-works/pi/issues/123))`
- External: `Added bar ([#456](https://github.com/earendil-works/pi/pull/456) by [@user](https://github.com/user))`

18
third_party/pi/.pi/prompts/dup.md vendored Normal file
View File

@@ -0,0 +1,18 @@
---
description: Find potential duplicates of a GitHub issue
argument-hint: "<issue-url>"
---
Look for potential duplicates of this issue in its GitHub repo: $1
Steps:
1. Read the issue with `gh issue view <url> --comments` to understand the problem.
2. Search open and closed issues and PRs in the same repo with `gh search issues` / `gh issue list --search` / `gh pr list --search`, using several keyword variations (error messages, feature names, affected components, symptoms).
3. Read promising candidates to confirm they describe the same problem, not just similar words.
A duplicate reports the same problem or requests the same change: same symptom with the same likely cause, or a PR that fixes this exact issue. Sharing a component, an error message, or a related cause is not enough.
Output a short draft GitHub comment with a writte by AI disclosure.
If there are none, reply "No duplicates found."
Do not post, label, or close anything.

21
third_party/pi/.pi/skills/github-work.md vendored Normal file
View File

@@ -0,0 +1,21 @@
---
name: github-work
description: Work on GitHub issues and pull requests of this repo. Use when asked to investigate, analyze, fix, or review an issue or PR, and when asked to wrap up such work (changelog, comment, commit, push, close).
---
# Working on GitHub issues and PRs
The repo's prompt templates in `.pi/prompts/` (two directories above this skill) define the workflow. Read the matching template in full and follow it. Where a template says `$ARGUMENTS` or `$@`, substitute the issue or PR number or URL from the request, and any extra instructions the user gave.
| Request | Template |
|---|---|
| Investigate, analyze, or fix an issue | `.pi/prompts/is.md` |
| Review a pull request | `.pi/prompts/pr.md` |
| Wrap up issue or PR work: changelog, final comment, commit, push, close | `.pi/prompts/wr.md` |
Rules:
- `is.md` analyzes and proposes only. Implement only when the user asks for it, then keep following AGENTS.md (checks, tests, changelog rules).
- `pr.md` never checks out or switches to the PR branch.
- Run `wr.md` only when the user asks to wrap up, commit, or push. Post at most one final GitHub comment per session, ending with the exact disclaimer line it specifies.
- When a request spans several steps, for example "fix issue X and wrap it up", read every template involved before starting.

19
third_party/pi/.pi/skills/interactive-testing.md vendored Normal file
View File

@@ -0,0 +1,19 @@
---
name: interactive-testing
description: Test and debug pi's interactive mode in a controlled tmux terminal. Use for TUI behavior checks and interactive release smoke tests.
---
# Testing pi Interactive Mode with tmux
Run the TUI in a controlled terminal (from the repo root, two directories above this skill):
```bash
tmux new-session -d -s pi-test -x 80 -y 24
tmux send-keys -t pi-test "./pi-test.sh" Enter
sleep 3 && tmux capture-pane -t pi-test -p # capture after startup
tmux send-keys -t pi-test "your prompt here" Enter
tmux send-keys -t pi-test Escape # special keys (also C-o for ctrl+o, etc.)
tmux kill-session -t pi-test
```
For release smoke tests, start the tmux session with `-c /tmp` and replace `./pi-test.sh` with the absolute path to the release binary. Test both Node and Bun binaries separately, submit a prompt, and wait for the model reply; startup alone is not a passing smoke test.

52
third_party/pi/.pi/skills/release.md vendored Normal file
View File

@@ -0,0 +1,52 @@
---
name: release
description: Prepare, publish, verify, and recover pi releases. Use for release preparation, local release smoke tests, publishing, and failed release CI or announcements.
---
# Releasing pi
Run repository commands from the repo root (two directories above this skill), unless instructed otherwise.
**Lockstep versioning**: all packages share one version; every release updates all together. `patch` = fixes + additions, `minor` = breaking changes. No major releases.
1. **Update CHANGELOGs**: ask the user whether they ran the `/cl` prompt on the latest commit on `main`. If not, they must run `/cl` first to audit and update each package's `[Unreleased]` section before releasing.
2. **Local smoke test**: build an unpublished release and smoke test from outside the repo (so it can't resolve workspace files):
```bash
npm run release:local -- --out /tmp/pi-local-release --force
cd /tmp
# Node package install smoke tests
/tmp/pi-local-release/node/pi --help
/tmp/pi-local-release/node/pi --version
/tmp/pi-local-release/node/pi --list-models
/tmp/pi-local-release/node/pi -p "Say exactly: ok"
/tmp/pi-local-release/node/pi
# Bun binary smoke tests
/tmp/pi-local-release/bun/pi --help
/tmp/pi-local-release/bun/pi --version
/tmp/pi-local-release/bun/pi --list-models
/tmp/pi-local-release/bun/pi -p "Say exactly: ok"
/tmp/pi-local-release/bun/pi
```
Verify both Node and Bun startup, model/account listing, interactive startup, and at least one real prompt with the intended default provider. The bare commands `/tmp/pi-local-release/node/pi` and `/tmp/pi-local-release/bun/pi` start interactive mode; run each in tmux, submit a prompt, and wait for the model reply before considering the interactive smoke test passed. Failures are release blockers unless the user explicitly accepts the risk.
Load and follow [interactive-testing.md](interactive-testing.md) for the tmux workflow. Start each release binary from `/tmp`, not the repo root.
3. **Run the release script**:
```bash
PI_ALLOW_LOCKFILE_CHANGE=1 npm_config_min_release_age=0 npm run release:patch # fixes + additions
PI_ALLOW_LOCKFILE_CHANGE=1 npm_config_min_release_age=0 npm run release:minor # breaking changes
```
If the test suite already passed in `npm run release:local` and only fails from machine load, `PI_RELEASE_SKIP_TESTS=1` skips `./test.sh` in the release script; checks and the packed install check still run. Use it only with the user's approval.
Use `npm_config_min_release_age=0` only for the release command. The repo's normal npm age gate can otherwise block the release lockfile refresh when the current workspace package version was published recently. Review any lockfile or install lock diffs the release creates before push.
The release script pins the newest published model catalog for Nix (`nix/model-catalog.json`) and builds the checkout with it as the Nix build does (it fails if that catalog predates main's model changes; publish one first), bumps all package versions, updates changelogs, regenerates release artifacts, runs `npm run check`, commits `Release vX.Y.Z`, tags `vX.Y.Z`, adds fresh `## [Unreleased]` changelog sections, commits `Add [Unreleased] section for next cycle`, then pushes `main` and the tag. Do not rerun the release script after a tag was pushed.
4. **CI verifies and announces the npm release**: pushing the `vX.Y.Z` tag triggers `.github/workflows/build-binaries.yml`. The `publish-npm` job uses npm trusted publishing through GitHub Actions OIDC with environment `npm-publish`; no local `npm publish`, `npm whoami`, OTP, or WebAuthn flow is required. After publishing, `announce-pi-dev-release` verifies every public workspace package resolves at the exact release version and that its npm tarball is available, then writes the verified release marker to R2. `pi.dev/api/latest-version` reads that marker; it must never announce a release from npm before this job succeeds.
5. **CI builds the Nix release**: the tag push also triggers `.github/workflows/nix.yml`, which builds the tagged flake on Linux and macOS and then fast-forwards the `stable` branch to the release commit. `nix run github:earendil-works/pi/stable` runs the latest release. If the Nix build fails, `stable` stays on the previous release; the tag cannot be fixed, so fix `main` and ship the fix in the next release.
6. **If CI publish or announcement fails**: inspect the failed job. The publish helper is idempotent and skips package versions already present on npm; the announcement job rechecks availability before updating the R2 marker. Rerun the failed job or workflow after fixing CI or transient npm issues. Do not rerun `npm run release:patch` or `npm run release:minor` for the same version.

58
third_party/pi/AGENTS.md vendored
View File

@@ -19,6 +19,7 @@
- Inline single-line helpers that have only one call site.
- Check node_modules for external API types; don't guess.
- **No inline imports** (`await import()`, `import("pkg").Type`, dynamic type imports). Top-level imports only.
- In `packages/coding-agent`, resolve package assets through helpers in `src/config.ts`. Do not use `__dirname` directly; the helpers account for source checkouts, npm installations, and standalone binaries.
- Never remove or downgrade code to fix type errors from outdated deps; upgrade the dep instead.
- Use only erasable TypeScript syntax (Node strip-only mode) in code checked by the root config (`packages/*/src`, `packages/*/test`, `packages/coding-agent/examples`): no parameter properties, `enum`, `namespace`/`module`, `import =`, `export =`, or other constructs needing JS emit. Use explicit fields with constructor assignments.
- Always ask before removing functionality or code that appears intentional.
@@ -41,11 +42,11 @@
## Dependency and Install Security
- Treat npm dep and lockfile changes as reviewed code. Direct external deps stay pinned to exact versions.
- Treat npm dep and lockfile changes as reviewed code. Direct registry deps, including internal workspace deps, stay pinned to exact versions.
- When updating `undici`, you MUST read its changelog/release notes for the target version and evaluate whether any changes may affect functionality before applying the update.
- Hydrate/update locally with `npm install --ignore-scripts`; clean/CI-style with `npm ci --ignore-scripts`. Don't run lifecycle scripts unless the user asks.
- If dep metadata changes, refresh `package-lock.json` with `npm install --package-lock-only --ignore-scripts`.
- If `packages/coding-agent/npm-shrinkwrap.json` needs regen, run `node scripts/generate-coding-agent-shrinkwrap.mjs` (verify with `--check` or `npm run check`). New deps with lifecycle scripts require review and an explicit allowlist entry in that script; never add one silently.
- If `packages/coding-agent/install-lock/` needs regen, run `node scripts/generate-coding-agent-install-lock.mjs` (verify with `--check` or `npm run check`). New deps with lifecycle scripts require review and an explicit allowlist entry in that script; never add one silently.
- Pre-commit blocks lockfile commits unless `PI_ALLOW_LOCKFILE_CHANGE=1`. Don't bypass unless the user wants the lockfile change committed.
## Git
@@ -96,16 +97,7 @@ When closing issues via commit:
## Testing pi Interactive Mode with tmux
Run the TUI in a controlled terminal (from the repo root):
```bash
tmux new-session -d -s pi-test -x 80 -y 24
tmux send-keys -t pi-test "./pi-test.sh" Enter
sleep 3 && tmux capture-pane -t pi-test -p # capture after startup
tmux send-keys -t pi-test "your prompt here" Enter
tmux send-keys -t pi-test Escape # special keys (also C-o for ctrl+o, etc.)
tmux kill-session -t pi-test
```
For testing pi's interactive mode, load and follow [.pi/skills/interactive-testing.md](.pi/skills/interactive-testing.md).
## Changelog
@@ -121,48 +113,12 @@ Rules:
Attribution:
- Internal (from issues): `Fixed foo bar ([#123](https://github.com/earendil-works/pi-mono/issues/123))`
- External contributions: `Added feature X ([#456](https://github.com/earendil-works/pi-mono/pull/456) by [@username](https://github.com/username))`
- Internal (from issues): `Fixed foo bar ([#123](https://github.com/earendil-works/pi/issues/123))`
- External contributions: `Added feature X ([#456](https://github.com/earendil-works/pi/pull/456) by [@username](https://github.com/username))`
## Releasing
**Lockstep versioning**: all packages share one version; every release updates all together. `patch` = fixes + additions, `minor` = breaking changes. No major releases.
1. **Update CHANGELOGs**: ask the user whether they ran the `/cl` prompt on the latest commit on `main`. If not, they must run `/cl` first to audit and update each package's `[Unreleased]` section before releasing.
2. **Local smoke test**: build an unpublished release and smoke test from outside the repo (so it can't resolve workspace files):
```bash
npm run release:local -- --out /tmp/pi-local-release --force
cd /tmp
# Node package install smoke tests
/tmp/pi-local-release/node/pi --help
/tmp/pi-local-release/node/pi --version
/tmp/pi-local-release/node/pi --list-models
/tmp/pi-local-release/node/pi -p "Say exactly: ok"
/tmp/pi-local-release/node/pi
# Bun binary smoke tests
/tmp/pi-local-release/bun/pi --help
/tmp/pi-local-release/bun/pi --version
/tmp/pi-local-release/bun/pi --list-models
/tmp/pi-local-release/bun/pi -p "Say exactly: ok"
/tmp/pi-local-release/bun/pi
```
Verify both Node and Bun startup, model/account listing, interactive startup, and at least one real prompt with the intended default provider. The bare commands `/tmp/pi-local-release/node/pi` and `/tmp/pi-local-release/bun/pi` start interactive mode; run each in tmux, submit a prompt, and wait for the model reply before considering the interactive smoke test passed. Failures are release blockers unless the user explicitly accepts the risk.
3. **Run the release script**:
```bash
PI_ALLOW_LOCKFILE_CHANGE=1 npm_config_min_release_age=0 npm run release:patch # fixes + additions
PI_ALLOW_LOCKFILE_CHANGE=1 npm_config_min_release_age=0 npm run release:minor # breaking changes
```
Use `npm_config_min_release_age=0` only for the release command. The repo's normal npm age gate can otherwise block the release lockfile refresh when the current workspace package version was published recently. Review any lockfile or shrinkwrap diffs the release creates before push.
The release script bumps all package versions, updates changelogs, regenerates release artifacts, runs `npm run check`, commits `Release vX.Y.Z`, tags `vX.Y.Z`, adds fresh `## [Unreleased]` changelog sections, commits `Add [Unreleased] section for next cycle`, then pushes `main` and the tag. Do not rerun the release script after a tag was pushed.
4. **CI verifies and announces the npm release**: pushing the `vX.Y.Z` tag triggers `.github/workflows/build-binaries.yml`. The `publish-npm` job uses npm trusted publishing through GitHub Actions OIDC with environment `npm-publish`; no local `npm publish`, `npm whoami`, OTP, or WebAuthn flow is required. After publishing, `announce-pi-dev-release` verifies every public workspace package resolves at the exact release version and that its npm tarball is available, then writes the verified release marker to R2. `pi.dev/api/latest-version` reads that marker; it must never announce a release from npm before this job succeeds.
5. **If CI publish or announcement fails**: inspect the failed job. The publish helper is idempotent and skips package versions already present on npm; the announcement job rechecks availability before updating the R2 marker. Rerun the failed job or workflow after fixing CI or transient npm issues. Do not rerun `npm run release:patch` or `npm run release:minor` for the same version.
For release preparation, publishing, verification, or recovery, load and follow [.pi/skills/release.md](.pi/skills/release.md).
## User Override

121
third_party/pi/README.md vendored
View File

@@ -10,26 +10,75 @@
> New issues and PRs from new contributors are auto-closed by default. Maintainers review auto-closed issues daily. See [CONTRIBUTING.md](CONTRIBUTING.md).
# Pi Agent Harness
# Pi
This is the home of the Pi agent harness project including our self extensible coding agent.
Pi is a minimal, extensible agent harness that you can make your own.
* **[@earendil-works/pi-coding-agent](packages/coding-agent)**: Interactive coding agent CLI
* **[@earendil-works/pi-agent-core](packages/agent)**: Agent runtime with tool calling and state management
* **[@earendil-works/pi-ai](packages/ai)**: Unified multi-provider LLM API (OpenAI, Anthropic, Google, …)
Adapt Pi to your workflows, not the other way around. Customize Pi with [extensions](packages/coding-agent/docs/extensions.md), [skills](packages/coding-agent/docs/skills.md), [prompt templates](packages/coding-agent/docs/prompt-templates.md), and [themes](packages/coding-agent/docs/themes.md). Bundle them as [Pi packages](packages/coding-agent/docs/packages.md) and share via npm or git.
To learn more about Pi:
Pi ships with powerful defaults but skips features like sub-agents and plan mode. Ask Pi to build what you want, or install a package that does it your way.
* [Visit pi.dev](https://pi.dev), the project website with demos
* [Read the documentation](https://pi.dev/docs/latest), but you can also ask the agent to explain itself
Use Pi [interactively](packages/coding-agent/docs/usage.md), automate it in [print or JSON mode](packages/coding-agent/docs/cli.md), control it over [RPC](packages/coding-agent/docs/rpc.md), or build apps with the [Pi TypeScript SDK](packages/coding-agent/docs/sdk.md). See [OpenClaw](https://github.com/OpenClaw/OpenClaw) for a real-world integration.
## All Packages
## Getting started
Install the command-line interface:
```bash
curl -fsSL https://pi.dev/install.sh | sh
```
On Windows:
```shell
powershell -c "irm https://pi.dev/install.ps1 | iex"
```
The installer pins all dependencies and updates Pi with `pi update`. Alternatively, install directly with npm, which does not pin transitive dependencies:
```bash
npm install -g --ignore-scripts @earendil-works/pi-coding-agent
```
Pi requires Node.js 22.19 or newer. The macOS, Linux, and Windows installers can install it if needed. Pi does not require dependency lifecycle scripts for a normal npm installation.
Start Pi in the directory where you want it to work:
```bash
cd /path/to/project
pi
```
For a built-in AI provider, run `/login` inside Pi to connect a subscription or API key. Then give Pi a task.
See the [documentation](https://pi.dev/docs/latest) for full setup and usage instructions, or [visit pi.dev](https://pi.dev) for demos.
## Run with Nix
```bash
nix run github:earendil-works/pi/stable
```
`stable` points at the latest release. Install it with `nix profile add github:earendil-works/pi/stable` and update with `nix profile upgrade pi`. Use a release tag such as `github:earendil-works/pi/v1.0.0` to pin a version, or `github:earendil-works/pi` for unreleased changes on `main`. Nix builds Pi from source.
Supports ARM64 and x86-64 on Linux and macOS. Use `nix build .` or `nix run .` to build or run your checkout.
Nix builds are offline, so the bundled model data comes from a pi.dev model catalog revision pinned in `nix/model-catalog.json`. At runtime, Pi still overlays newer catalog data from pi.dev as usual. The Nix workflow replaces the pin on `main` when it no longer matches the checkout, for example after a provider is added or gains a new model type. To refresh it by hand:
```bash
npm run update:model-catalog-pin
```
## Packages
This monorepo contains the Pi CLI and its supporting libraries.
| Package | Description |
|---------|-------------|
| **[@earendil-works/chord](packages/chord)** | Standalone application-composition runtime for services, replicated state, RPC, and plugins |
| **[@earendil-works/chord](packages/chord)** | Typed host runtime for hot-reloadable plugins, contracts, replicated state, and RPC over JSON links |
| **[@earendil-works/pi-telemetry](packages/telemetry)** | Vendor-neutral telemetry contracts, reference adapter, conformance tests, and typed schemas |
| **[@earendil-works/pi-ai](packages/ai)** | Unified multi-provider LLM API (OpenAI, Anthropic, Google, etc.) |
| **[@earendil-works/pi-durable](packages/durable)** | Durable conversation, task, and document runtime |
| **[@earendil-works/pi-agent-core](packages/agent)** | Agent runtime with tool calling and state management |
| **[@earendil-works/pi-coding-agent](packages/coding-agent)** | Interactive coding agent CLI |
| **[@earendil-works/pi-tui](packages/tui)** | Terminal UI library with differential rendering |
@@ -61,6 +110,48 @@ npm run check # Lint, format, and type check
./pi-test.sh # Run pi from sources (can be run from any directory)
```
### Using local packages outside the monorepo
Build every public package into one coherent local artifact set:
```bash
npm run pack:packages -- --out .artifacts/pi-packages
```
This refreshes model data before building `pi-ai`. To avoid network access when
model data is already hydrated, pass `--offline-model-data`.
Then configure an external project to consume one package and resolve all of
its Pi dependencies from the same artifact set. npm is the default:
```bash
node scripts/use-local-packages.mjs \
--manifest .artifacts/pi-packages/manifest.json \
--consumer ../my-project \
--package @earendil-works/pi-durable \
--package @earendil-works/pi-agent-core
cd ../my-project
npm install --ignore-scripts
```
For a pnpm project, point `--consumer` at the workspace root:
```bash
node scripts/use-local-packages.mjs \
--manifest .artifacts/pi-packages/manifest.json \
--consumer ../my-project \
--package @earendil-works/pi-agent-core \
--package-manager pnpm
cd ../my-project
pnpm install --ignore-scripts
```
Repeat `--package` for each direct dependency. The command updates the
consumer's `package.json` with content-addressed local `file:` references. It
writes transitive overrides to `package.json` for npm or `pnpm-workspace.yaml`
for pnpm. Keep the artifact directory available while installing or updating
the consumer. Re-run both commands after changing Pi source.
## Building standalone binaries from release source
GitHub releases include a versioned source archive covered by the release's `SHA256SUMS` file. Extract it and run the same build script used for the official standalone binaries:
@@ -72,7 +163,7 @@ cd "pi-${VERSION}"
./scripts/build-binaries.sh --offline-model-data --platform linux-x64 --out "$PWD/out"
```
The source archive includes the generated provider model data used for the release. `--offline-model-data` builds with that snapshot instead of refreshing it from live provider catalogs. The script still installs dependencies, builds the monorepo, compiles the Bun executable, and stages its runtime assets. Package maintainers who provide dependencies separately can pass `--skip-install --skip-deps`.
The archive includes release model data and native prebuilds. `--offline-model-data` uses that model data without refreshing provider catalogs. The script installs dependencies and builds the executable with its runtime assets; pass `--skip-install` if dependencies are already provided.
## Supply-chain hardening
@@ -81,12 +172,12 @@ We treat npm dependency changes as reviewed code changes.
- Direct external dependencies are pinned to exact versions. Internal workspace packages remain version-ranged.
- `.npmrc` sets `save-exact=true` and `min-release-age=2` to avoid same-day dependency releases during npm resolution.
- `package-lock.json` is the dependency ground truth. Pre-commit blocks accidental lockfile commits unless `PI_ALLOW_LOCKFILE_CHANGE=1` is set.
- `npm run check` verifies pinned direct deps, native TypeScript import compatibility, and the generated coding-agent shrinkwrap.
- The published CLI package includes `packages/coding-agent/npm-shrinkwrap.json`, generated from the root lockfile, to pin transitive deps for npm users.
- Release smoke tests use `npm run release:local` to build, pack, and create isolated npm and Bun installs outside the repo before tagging a release.
- `npm run check` verifies pinned direct deps, native TypeScript import compatibility, and the generated coding-agent install lock.
- The pi.dev installer installs from `packages/coding-agent/install-lock/`, generated from the root lockfile, to pin transitive deps. The npm package does not pin transitive deps.
- Local release smoke tests and npm publication use the same tarball packer; npm publishes the validated tarballs rather than repacking workspace directories.
- Local release installs, documented npm installs, and `pi update --self` use `--ignore-scripts` where supported.
- CI installs with `npm ci --ignore-scripts`, and a scheduled GitHub workflow runs `npm audit --omit=dev` plus `npm audit signatures --omit=dev`.
- Shrinkwrap generation has an explicit allowlist for dependency lifecycle scripts; new lifecycle-script deps fail checks until reviewed.
- Install lock generation has an explicit allowlist for dependency lifecycle scripts; new lifecycle-script deps fail checks until reviewed.
## Share your OSS coding agent sessions

11
third_party/pi/biome.json vendored
View File

@@ -25,17 +25,20 @@
},
"files": {
"includes": [
"scripts/model-catalog-protocol.ts",
"scripts/model-catalog-protocol.test.ts",
"packages/*/src/**/*.ts",
"packages/*/test/**/*.ts",
"packages/session-backends/*/src/**/*.ts",
"packages/session-backends/*/test/**/*.ts",
"packages/coding-agent/examples/**/*.ts",
"packages/agent/examples/**/*.ts",
"!**/node_modules/**/*",
"!**/test-sessions.ts",
"!**/models.generated.ts",
"!**/*.models.ts",
"!packages/mom/data/**/*",
"!!**/node_modules"
"!!**/node_modules",
"!!apps"
]
}
},
"plugins": ["./scripts/biome/model-type-comparison.grit"]
}

44
third_party/pi/flake.lock generated vendored Normal file
View File

@@ -0,0 +1,44 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1788316716,
"narHash": "sha256-bc7rSpXIdn9QWGNqfWcPZWOhEVF8NoeAZkWq0XWnf/k=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "3ed67ec0a4d3c7ab4ae1f04f8ee8df07bfa506a2",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-darwin-x64": {
"locked": {
"lastModified": 1788334797,
"narHash": "sha256-nskSa8kDimM7F0VFSP8IdoSNo4G0vKzOtxwvOFpoGDI=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "c19db427a1fdfc7591c0b0baeb4665dcef2c61da",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-26.05-darwin",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs",
"nixpkgs-darwin-x64": "nixpkgs-darwin-x64"
}
}
},
"root": "root",
"version": 7
}

54
third_party/pi/flake.nix vendored Normal file
View File

@@ -0,0 +1,54 @@
{
description = "Pi coding agent";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
# nixpkgs unstable no longer supports Intel macOS. Keep using the final
# Darwin branch that does so for pi's x86_64-darwin package.
nixpkgs-darwin-x64.url = "github:NixOS/nixpkgs/nixpkgs-26.05-darwin";
};
outputs =
{
self,
nixpkgs,
nixpkgs-darwin-x64,
}:
let
systems = [
"aarch64-darwin"
"aarch64-linux"
"x86_64-darwin"
"x86_64-linux"
];
forAllSystems = nixpkgs.lib.genAttrs systems;
nixpkgsFor = system: if system == "x86_64-darwin" then nixpkgs-darwin-x64 else nixpkgs;
packageFor =
system:
let
pkgs = import (nixpkgsFor system) { inherit system; };
in
pkgs.callPackage ./nix/package.nix { source = self; };
in
{
packages = forAllSystems (system: {
default = packageFor system;
pi = packageFor system;
});
apps = forAllSystems (system: {
default = {
type = "app";
program = "${self.packages.${system}.default}/bin/pi";
meta.description = "Pi coding agent";
};
pi = self.apps.${system}.default;
});
overlays.default = final: _previous: {
pi = final.callPackage ./nix/package.nix { source = self; };
};
};
}

46
third_party/pi/mini-test.sh vendored
View File

@@ -1,46 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MINI_SRC="$SCRIPT_DIR/packages/coding-agent/src/experimental/mini/main.ts"
MINI_DIST="$SCRIPT_DIR/packages/coding-agent/dist/experimental/mini/main.js"
# The session server is spawned detached and outlives the TUI, so it keeps running whatever code it
# started with. Restart it after changing anything under mini/, or the protocol will not match.
stop_server() {
pkill -f "mini/server/entry" 2>/dev/null || true
rm -f "${PI_AGENT_DIR:-$HOME/.pi/agent}/experimental/mini.sock"
}
USE_DIST=false
ARGS=()
for arg in "$@"; do
case "$arg" in
--dist) USE_DIST=true ;;
--fresh) stop_server ;;
--stop) stop_server; echo "Stopped the mini session server."; exit 0 ;;
--help)
cat <<'USAGE'
Usage: ./mini-test.sh [--dist] [--fresh] [--stop] [mini args...]
--dist run built output with plain node instead of tsx on sources
--fresh stop the detached session server first, so it picks up your changes
--stop stop the detached session server and exit
Mini args are passed through, for example:
./mini-test.sh --continue
USAGE
exit 0 ;;
*) ARGS+=("$arg") ;;
esac
done
if [[ "$USE_DIST" == "true" ]]; then
if [[ ! -f "$MINI_DIST" ]]; then
echo "No build found. Run: npm run build -w @earendil-works/pi-coding-agent" >&2
exit 1
fi
exec node "$MINI_DIST" ${ARGS[@]+"${ARGS[@]}"}
fi
exec "$SCRIPT_DIR/node_modules/.bin/tsx" --tsconfig "$SCRIPT_DIR/tsconfig.json" "$MINI_SRC" ${ARGS[@]+"${ARGS[@]}"}

3
third_party/pi/nix/model-catalog.json vendored Normal file
View File

@@ -0,0 +1,3 @@
{
"revision": "sha256-ffa3e98626230f378f80200debb736c4a2238b5f2ba246a92cfc7d8e19604678"
}

180
third_party/pi/nix/package.nix vendored Normal file
View File

@@ -0,0 +1,180 @@
{
autoPatchelfHook,
fd,
fetchurl,
importNpmLock,
lib,
libxcb,
makeWrapper,
nodejs_22,
ripgrep,
source,
stdenv,
wl-clipboard,
xclip,
}:
let
nodejs = nodejs_22;
packageJson = lib.importJSON (source + "/packages/coding-agent/package.json");
# Lockfile root used by the pi.dev installer. It pins the coding agent's
# runtime dependency tree and is kept in sync with package-lock.json by
# `npm run check`.
installLock = source + "/packages/coding-agent/install-lock";
modelCatalogPin = lib.importJSON ./model-catalog.json;
modelCatalog = fetchurl {
name = "pi-model-catalog.json";
# The typed catalog is the representation whose bytes the revision hashes.
url = "https://pi.dev/api/models/revisions/${modelCatalogPin.revision}?types=chat,image,classifier";
sha256 = lib.removePrefix "sha256-" modelCatalogPin.revision;
};
workspacePackages = stdenv.mkDerivation {
pname = "pi-workspace-packages";
inherit (packageJson) version;
src = source;
npmDeps = importNpmLock { npmRoot = source; };
npmRebuildFlags = [ "--ignore-scripts" ];
nativeBuildInputs = [
nodejs
importNpmLock.npmConfigHook
];
buildPhase = ''
runHook preBuild
node packages/ai/scripts/hydrate-model-catalog.ts ${modelCatalog}
npm run build:offline
runHook postBuild
'';
installPhase = ''
runHook preInstall
pack_package() {
local package_dir="$1"
local output_name="$2"
local tarball
tarball="$(cd "$package_dir" && npm pack --ignore-scripts --silent --pack-destination "$TMPDIR")"
mv "$TMPDIR/$tarball" "$out/$output_name.tgz"
}
mkdir -p "$out"
pack_package packages/telemetry telemetry
pack_package packages/ai ai
pack_package packages/tui tui
pack_package packages/agent agent
pack_package packages/codemode codemode
pack_package packages/mcp mcp
pack_package packages/coding-agent coding-agent
runHook postInstall
'';
};
npmDeps = importNpmLock {
npmRoot = installLock;
# The install lock points internal packages at registry releases. Replace
# them with the packages built from this checkout.
packageSourceOverrides = {
"node_modules/@earendil-works/pi-agent-core" = workspacePackages + "/agent.tgz";
"node_modules/@earendil-works/pi-ai" = workspacePackages + "/ai.tgz";
"node_modules/@earendil-works/pi-codemode" = workspacePackages + "/codemode.tgz";
"node_modules/@earendil-works/pi-coding-agent" = workspacePackages + "/coding-agent.tgz";
"node_modules/@earendil-works/pi-mcp" = workspacePackages + "/mcp.tgz";
"node_modules/@earendil-works/pi-telemetry" = workspacePackages + "/telemetry.tgz";
"node_modules/@earendil-works/pi-tui" = workspacePackages + "/tui.tgz";
};
};
in
stdenv.mkDerivation {
pname = "pi";
inherit (packageJson) version;
src = installLock;
inherit npmDeps;
npmRebuildFlags = [ "--ignore-scripts" ];
nativeBuildInputs = [
nodejs
importNpmLock.npmConfigHook
makeWrapper
]
++ lib.optionals stdenv.hostPlatform.isLinux [ autoPatchelfHook ];
buildInputs = [ nodejs ] ++ lib.optionals stdenv.hostPlatform.isLinux [
stdenv.cc.cc.lib
libxcb
];
dontBuild = true;
dontStrip = true;
installPhase = ''
runHook preInstall
mkdir -p "$out/lib/pi" "$out/bin"
cp -R . "$out/lib/pi"
# The wrapper runs Pi with an absolute Node path. Node is only on PATH so Pi can
# find npm for package installs; append it so that shells started by the
# bash tool keep the user's node/npm (e.g. from mise or nvm). fd, ripgrep,
# and the clipboard tools are Pi's own dependencies and take precedence.
makeWrapper ${nodejs}/bin/node "$out/bin/pi" \
--add-flags "$out/lib/pi/node_modules/@earendil-works/pi-coding-agent/dist/bundle/cli.js" \
--suffix PATH : ${lib.makeBinPath [ nodejs ]} \
--prefix PATH : ${
lib.makeBinPath (
[
fd
ripgrep
]
++ lib.optionals stdenv.hostPlatform.isLinux [
wl-clipboard
xclip
]
)
}
runHook postInstall
'';
doInstallCheck = true;
installCheckPhase = ''
runHook preInstallCheck
test "$("$out/bin/pi" --version)" = "${packageJson.version}"
# Load host-platform TUI helpers directly so missing native dependencies
# fail the build rather than silently disabling clipboard support.
${nodejs}/bin/node -e \
"const fs = require('node:fs');
const path = require('node:path');
const dir = '$out/lib/pi/node_modules/@earendil-works/pi-tui/native/' + process.platform + '/prebuilds/' + process.platform + '-' + process.arch;
if (fs.existsSync(dir)) {
for (const file of fs.readdirSync(dir)) {
if (file.endsWith('.node')) require(path.join(dir, file));
}
}"
${nodejs}/bin/node -e \
"require('$out/lib/pi/node_modules/@silvia-odwyer/photon-node')"
runHook postInstallCheck
'';
meta = {
description = packageJson.description;
homepage = "https://pi.dev";
license = lib.licenses.mit;
mainProgram = "pi";
platforms = [
"aarch64-darwin"
"aarch64-linux"
"x86_64-darwin"
"x86_64-linux"
];
sourceProvenance = with lib.sourceTypes; [
fromSource
binaryNativeCode
];
};
}

2962
third_party/pi/package-lock.json generated vendored
View File

File diff suppressed because it is too large Load Diff

39
third_party/pi/package.json vendored
View File

@@ -4,7 +4,7 @@
"type": "module",
"workspaces": [
"packages/*",
"packages/session-backends/*",
"apps/*",
"packages/coding-agent/examples/extensions/with-deps",
"packages/coding-agent/examples/extensions/custom-provider-anthropic",
"packages/coding-agent/examples/extensions/custom-provider-gitlab-duo",
@@ -13,28 +13,32 @@
],
"scripts": {
"clean": "npm run clean --workspaces",
"build": "cd packages/chord && npm run build && cd ../tui && npm run build && cd ../telemetry && npm run build && cd ../ai && npm run build && cd ../agent && npm run build && cd ../session-backends/sqlite-node && npm run build && cd ../../protocol && npm run build && cd ../client && npm run build && cd ../server && npm run build && cd ../coding-agent && npm run build",
"build:offline": "cd packages/chord && npm run build && cd ../tui && npm run build && cd ../telemetry && npm run build && cd ../ai && npm run build:offline && cd ../agent && npm run build && cd ../session-backends/sqlite-node && npm run build && cd ../../protocol && npm run build && cd ../client && npm run build && cd ../server && npm run build && cd ../coding-agent && npm run build",
"check": "biome check --write --error-on-warnings . && npm run check:pinned-deps && npm run check:runtime-deps && npm run check:ts-imports && npm run check:entry-graphs && npm run check:shrinkwrap && npm run check:install-lock:coding-agent && tsgo --noEmit && npm run check:browser-smoke",
"build": "cd packages/chord && npm run build && cd ../tui && npm run build && cd ../telemetry && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build",
"build:offline": "cd packages/chord && npm run build && cd ../tui && npm run build && cd ../telemetry && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build:offline && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build",
"build:native:darwin": "npm --prefix packages/tui run build:native:darwin",
"build:native:linux": "npm --prefix packages/tui run build:native:linux",
"build:native:win32": "npm --prefix packages/tui run build:native:win32",
"check": "node scripts/biome-check.mjs --write && npm run check:pinned-deps && npm run check:runtime-deps && npm run check:ts-imports && npm run check:entry-graphs && npm run check:install-lock:coding-agent && tsc --noEmit && npm run check:browser-smoke",
"check:browser-smoke": "node scripts/check-browser-smoke.mjs",
"check:pinned-deps": "node scripts/check-pinned-deps.mjs",
"check:runtime-deps": "node scripts/check-runtime-deps.mjs",
"check:package-install": "node scripts/coding-agent-consumer.mjs",
"check:shrinkwrap": "node scripts/generate-coding-agent-shrinkwrap.mjs --check",
"check:package-install": "node scripts/check-package-install.mjs",
"check:install-lock:coding-agent": "node scripts/generate-coding-agent-install-lock.mjs --check",
"check:ts-imports": "node scripts/check-ts-relative-imports.mjs",
"check:entry-graphs": "node scripts/check-entry-graphs.mjs",
"generate:models": "npm --prefix packages/ai run generate-models && npm --prefix packages/ai run generate-image-models",
"generate:models": "npm --prefix packages/ai run generate-models",
"hydrate:model-data": "npm --prefix packages/ai run hydrate-model-data",
"check:model-data": "npm --prefix packages/ai run check:model-data",
"generate:model-catalog": "npm --prefix packages/ai run generate-model-catalog",
"diff:model-catalog": "node scripts/diff-model-catalog.mjs",
"update:model-catalog-pin": "node scripts/update-model-catalog-pin.mjs",
"eval": "npm run eval --workspace=@earendil-works/pi-evals --",
"check:model-catalog": "node scripts/publish-model-catalog.mjs --input .artifacts/model-catalog --dry-run",
"profile:tui": "node scripts/profile-coding-agent-node.mjs --mode tui",
"profile:rpc": "node scripts/profile-coding-agent-node.mjs --mode rpc",
"test": "npm run test:scripts && npm run test --workspaces --if-present",
"test:scripts": "node --test scripts/*.test.mjs",
"test:scripts": "node --test scripts/*.test.mjs scripts/*.test.ts",
"test:mcp-conformance": "node packages/coding-agent/test/mcp-conformance/run.ts",
"version:patch": "npm version patch --workspaces --no-git-tag-version --no-workspaces-update && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts",
"version:minor": "npm version minor --workspaces --no-git-tag-version --no-workspaces-update && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts",
"version:major": "npm version major --workspaces --no-git-tag-version --no-workspaces-update && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts",
@@ -43,7 +47,7 @@
"publish": "npm run prepublishOnly && node scripts/publish.mjs",
"publish:dry": "npm run prepublishOnly && node scripts/publish.mjs --dry-run",
"release:local": "node scripts/local-release.mjs",
"shrinkwrap:coding-agent": "node scripts/generate-coding-agent-shrinkwrap.mjs",
"pack:packages": "node scripts/pack-packages.mjs",
"install-lock:coding-agent": "node scripts/generate-coding-agent-install-lock.mjs",
"release:patch": "node scripts/release.mjs patch",
"release:minor": "node scripts/release.mjs minor",
@@ -54,23 +58,20 @@
"devDependencies": {
"@anthropic-ai/sandbox-runtime": "0.0.26",
"@biomejs/biome": "2.3.5",
"@biomejs/wasm-nodejs": "2.3.5",
"@types/node": "22.19.19",
"@typescript/native-preview": "7.0.0-dev.20260120.1",
"esbuild": "0.28.1",
"esbuild": "0.28.2",
"husky": "9.1.7",
"shx": "0.4.0",
"tsx": "4.22.1",
"typescript": "5.9.3"
"pnpm": "11.15.1",
"shx": "0.3.4",
"typescript": "7.0.2",
"yaml": "2.9.0"
},
"engines": {
"node": ">=22.19.0"
},
"version": "0.0.3",
"overrides": {
"protobufjs": "7.6.5",
"rimraf": "6.1.2",
"gaxios": {
"rimraf": "6.1.2"
}
"protobufjs": "7.6.6"
}
}

75
third_party/pi/packages/agent/CHANGELOG.md vendored
View File

@@ -1,5 +1,80 @@
# Changelog
## [1.3.0] - 2026-10-15
## [1.2.0] - 2026-10-12
## [1.1.0] - 2026-10-07
### Added
- Added `durationMs` to tool result messages, `AgentToolCallOutcome`, and `tool_execution_end` events: how long `execute()` took, measured with a monotonic clock and excluding hooks. Calls that did not run have none ([#10549](https://github.com/earendil-works/pi/issues/10549))
### Changed
- `streamProxy()` returns an `AssistantMessageEventStream`, so proxied responses get `durationMs` like direct ones
## [1.0.4] - 2026-10-05
## [1.0.3] - 2026-10-05
## [1.0.2] - 2026-10-04
## [1.0.1] - 2026-10-03
## [1.0.0] - 2026-10-01
### Breaking Changes
- Removed the experimental harness from `@earendil-works/pi-agent-core`: `AgentHarness`, sessions and session storage, the durable runtime, pico3, harness tools, compaction, skills, prompt templates, system prompt helpers, telemetry schemas, the search service types, and the `uuidv7` and pi-telemetry re-exports. The `./node`, `./harness/*`, and `./experimental/pico3` subpath exports are gone. The package now contains only `Agent`, the agent loop, the proxy stream, and their types. Use `@earendil-works/pi-durable` for durable sessions.
## [0.99.2] - 2026-09-30
## [0.99.1] - 2026-09-29
## [0.99.0] - 2026-09-29
### Added
- Added the `onProviderStreamEvent` agent option, which is passed to provider streams to observe parsed provider events before normalization ([#9784](https://github.com/earendil-works/pi/issues/9784), [#9901](https://github.com/earendil-works/pi/pull/9901) by [@davidbrai](https://github.com/davidbrai)).
- The agent loop now records the requested thinking level as `thinkingLevel` on each assistant message.
## [0.87.1] - 2026-09-22
## [0.87.0] - 2026-09-21
### Breaking Changes
- Removed `AgentOptions.shouldStopAfterTurn` and `AgentLoopConfig.shouldStopAfterTurn`. Use `finishTurn` and return `{ action: "end" }` to stop after the completed turn:
```ts
// Before
shouldStopAfterTurn: async (turn, signal) => await shouldStop(turn, signal),
// After
finishTurn: async (turn, signal) => {
// shouldStopAfterTurn previously ran only for normal responses.
if (turn.message.stopReason === "error" || turn.message.stopReason === "aborted") return;
return (await shouldStop(turn, signal)) ? { action: "end" } : undefined;
},
```
`finishTurn` runs after the assistant and all tool results are finalized but before `turn_end`; its decision is applied after `turn_end`. It also runs for error and aborted responses, whose decisions are ignored because those responses remain hard exits. The guard in the migration preserves the old hook's normal-response-only invocation, including avoiding predicate side effects on hard exits. Returning `{ action: "end" }` leaves steering and follow-up queues untouched and skips `prepareNextTurn`.
### Added
- Added `prepareRequest`, which runs before every provider request, including the first. For example, return `{ context: { ...context, messages: persistedMessages } }` to install canonical context after already-selected input is emitted without introducing another queue poll.
- Added `finishTurn`, which runs after assistant/tool-result finalization and before `turn_end` for normal, error, and aborted responses. Return `{ action: "end" }` to end a normal run after `turn_end`, or `undefined` to preserve normal scheduling. `{ action: "continue" }` ensures one next provider request: existing tool-result, steering, or follow-up scheduling can satisfy that request without adding another one; otherwise the loop makes one context-only request. Error and aborted responses remain hard exits.
- Added `Agent.peekQueuedMessages()` to preview the next queue-selected batch without consuming it.
### Fixed
- Fixed harness reads misclassifying text files beginning with `GIF` as images ([#9755](https://github.com/earendil-works/pi/issues/9755)).
## [0.86.1] - 2026-09-20
## [0.86.0] - 2026-09-19
## [0.85.1] - 2026-09-05
## [0.85.0] - 2026-09-04

135
third_party/pi/packages/agent/README.md vendored
View File

@@ -8,10 +8,6 @@ Stateful agent with tool execution and event streaming. Built on `@earendil-work
npm install @earendil-works/pi-agent-core
```
### SQLite session backends
The SQLite session backend and the `node:sqlite` adapter live in a separate package, `@earendil-works/pi-session-backend-sqlite-node`, so the core package does not pull in runtime builtins or native SQLite dependencies by default. The backend accepts a runtime-specific SQLite factory, allowing other session backends to ship as their own packages in the future.
## Quick Start
```typescript
@@ -42,10 +38,6 @@ agent.subscribe((event) => {
await agent.prompt("Hello!");
```
## Experimental facet services
Transport-neutral facet-service primitives live in `@earendil-works/chord`. The agent core does not export the service runtime.
## Core Concepts
### AgentMessage vs LLM Message
@@ -127,38 +119,68 @@ The `beforeToolCall` hook runs after `tool_execution_start` and validated argume
Tools, blocked `beforeToolCall` results, and `afterToolCall` overrides can return `terminate: true` to hint that the automatic follow-up LLM call should be skipped. The loop only stops early when every finalized tool result in that batch sets `terminate: true`. Mixed batches continue normally.
The `Agent` class accepts `shouldStopAfterTurn` in `AgentOptions`. Low-level loop callers can set the same hook in `AgentLoopConfig`:
```typescript
const stream = agentLoop(
prompts,
context,
{
model,
convertToLlm,
shouldStopAfterTurn: async ({ message, toolResults, context, newMessages }) => {
return shouldCompactBeforeNextTurn(context.messages);
},
},
undefined,
models.streamSimple.bind(models),
);
```
`shouldStopAfterTurn` runs after `turn_end` is emitted and after the assistant response and any tool executions have completed normally. If it returns `true`, the loop emits `agent_end` and exits before polling steering or follow-up queues, and before starting another LLM call. It does not abort the provider stream, does not cancel running tools, and does not alter the assistant message stop reason. The `AgentOptions` callback also receives the active run's `AbortSignal` as its second argument.
When you use the `Agent` class, assistant `message_end` processing is treated as a barrier before tool preflight begins. That means `beforeToolCall` sees agent state that already includes the assistant message that requested the tool call.
### continue() Event Sequence
### Request preparation and turn finalization
`continue()` resumes from existing context without adding a new message. Use it for retries after errors.
`prepareRequest` runs immediately before every conversational provider request, including the first. Use it to install canonical persisted context after pending input has been emitted:
```typescript
// After an error, retry from current state
await agent.continue();
agent.prepareRequest = async ({ context }) => ({
context: { ...context, messages: await session.loadModelContext() },
});
```
The last message in context must be `user` or `toolResult` (not `assistant`).
`prepareRequest` does not poll queues. Steering queued while it runs waits for the next normal steering poll.
`finishTurn` runs after the assistant and all tool results are finalized, but before `turn_end`. It runs for normal, error, and aborted responses:
```typescript
agent.finishTurn = async ({ message }) => {
if (message.stopReason === "error" || message.stopReason === "aborted") return;
if (shouldEndRun(message)) return { action: "end" };
return needsAnotherResponse(message) ? { action: "continue" } : undefined;
};
```
Returning `undefined` preserves normal scheduling. `{ action: "end" }` stops immediately after `turn_end`, before polling steering or follow-up queues or preparing another request. On a normal response, `{ action: "continue" }` ensures one next provider request. If tool results, steering, or a follow-up already cause that request, they satisfy the decision and no additional request is made; otherwise the loop makes one context-only request. Error and aborted responses remain hard exits, so their decisions are ignored. `finishTurn` runs again after the next request, so returning `{ action: "continue" }` unconditionally creates an endless loop.
To migrate from the removed `shouldStopAfterTurn`, return `{ action: "end" }`. Guard error and aborted responses to preserve the old hook's normal-response-only invocation, especially when the predicate has side effects or assumes a successful response:
```typescript
finishTurn: async (turn, signal) => {
if (turn.message.stopReason === "error" || turn.message.stopReason === "aborted") return;
return (await shouldStop(turn, signal)) ? { action: "end" } : undefined;
},
```
Each provider turn follows this lifecycle:
```text
selected input events
→ prepareRequest
→ provider response
→ tool results
→ finishTurn
→ turn_end
→ existing continuation scheduling or agent_end
```
### continue() and queued input
`continue()` retains its existing queue behavior. Empty and system-only transcripts reject without consuming queues. A non-assistant tail continues from existing context: steering is polled at startup, while follow-up input waits until the response naturally stops.
```typescript
agent.followUp({ role: "user", content: "After the retry", timestamp: Date.now() });
await agent.continue(); // The first request retries the existing user/toolResult tail.
```
An assistant tail cannot be sent directly, so `continue()` falls back to one queued steering batch, then one queued follow-up batch. Queue mode still controls whether that selected batch contains one message or all messages:
```typescript
agent.steer({ role: "user", content: "Continue from here", timestamp: Date.now() });
await agent.continue(); // Uses the queued message only because the tail is assistant.
```
### Event Types
@@ -181,7 +203,8 @@ The last message in context must be `user` or `toolResult` (not `assistant`).
```typescript
const agent = new Agent({
// Initial state
// Initial state. systemPrompt and tools become the leading system message
// unless messages already starts with one.
initialState: {
systemPrompt: string,
model: Model<any>,
@@ -202,7 +225,8 @@ const agent = new Agent({
// Follow-up mode: "one-at-a-time" (default) or "all"
followUpMode: "one-at-a-time",
// Required stream function
// Required stream function. Receives a TranscriptContext: the prompt and tools
// are in the transcript's system messages, not on the context.
streamFn: models.streamSimple.bind(models),
// Session ID for provider caching
@@ -231,9 +255,16 @@ const agent = new Agent({
}
},
// Stop gracefully after a completed turn, before queued messages are polled.
shouldStopAfterTurn: async ({ context }, signal) => {
return shouldCompactBeforeNextTurn(context.messages, signal);
// Rebuild finalized context immediately before every provider request.
prepareRequest: async ({ context }, signal) => {
return { context: { ...context, messages: await loadCanonicalMessages(signal) } };
},
// Finalize a completed turn before turn_end is emitted.
// `continue` ensures one next request; existing tool/queue scheduling can satisfy it.
// `end` ends this run after turn_end without polling queues.
finishTurn: async ({ message, toolResults }, signal) => {
return shouldContinue(message, toolResults) ? { action: "continue" } : undefined;
},
// Custom thinking budgets for token-based providers
@@ -250,7 +281,6 @@ const agent = new Agent({
```typescript
interface AgentState {
systemPrompt: string;
model: Model<any>;
thinkingLevel: ThinkingLevel;
tools: AgentTool<any>[];
@@ -266,6 +296,17 @@ Access state via `agent.state`.
Assigning `agent.state.tools = [...]` or `agent.state.messages = [...]` copies the top-level array before storing it. Mutating the returned array mutates the current agent state.
The transcript owns the system prompt and tool declarations: the leading system message is the prompt, later system messages patch it (see `SystemMessage` in pi-ai). `agent.state.systemPrompt` is read-only and replays the transcript. `agent.state.tools` is the executable loadout; before every request the loop diffs it against the tools the transcript declares and, if they differ, announces the change in a system message (merged into a pending system message when one exists). pi-ai's `getCurrentSystemMessage(messages)` returns the replayed head, including declared tools, for any message array, including agent transcripts with custom message roles.
To change the prompt mid-conversation, append a system message with `content` (added instructions) or `sections` (named replacements):
```typescript
await agent.prompt([
{ role: "system", content: "", sections: { skills: "<skills>...</skills>" }, timestamp: Date.now() },
{ role: "user", content: "Continue", timestamp: Date.now() },
]);
```
During streaming, `agent.state.streamingMessage` contains the current partial assistant message.
`agent.state.isStreaming` remains `true` until the run fully settles, including awaited `agent_end` subscribers.
@@ -286,23 +327,26 @@ await agent.prompt("What's in this image?", [
// AgentMessage directly
await agent.prompt({ role: "user", content: "Hello", timestamp: Date.now() });
// Continue from current context (last message must be user or toolResult)
// Continue existing non-assistant input; an assistant tail may use queued input as fallback
await agent.continue();
```
### State Management
```typescript
agent.state.systemPrompt = "New prompt";
agent.state.model = getModel("openai", "gpt-4o");
agent.state.thinkingLevel = "medium";
agent.state.tools = [myTool];
agent.toolExecution = "sequential";
agent.beforeToolCall = async ({ toolCall }) => undefined;
agent.afterToolCall = async ({ toolCall, result }) => undefined;
agent.shouldStopAfterTurn = async ({ context }) => shouldCompactBeforeNextTurn(context.messages);
agent.prepareRequest = async ({ context }) => ({
context: { ...context, messages: await loadCanonicalMessages() },
});
agent.finishTurn = async () => undefined;
agent.state.messages = newMessages; // top-level array is copied
agent.state.messages.push(message);
const nextQueuedMessages = agent.peekQueuedMessages(); // respects queue modes; does not consume
agent.reset();
```
@@ -459,6 +503,10 @@ Thrown errors are caught by the agent and reported to the LLM as tool errors wit
Return `terminate: true` from `execute()`, a blocked `beforeToolCall`, or `afterToolCall` to hint that the agent should stop after the current tool batch. This only takes effect when every finalized tool result in the batch is terminating. The hint is runtime-only; emitted `toolResult` transcript messages remain standard LLM tool results.
### MCP and Codemode
`@earendil-works/pi-mcp` connects to MCP servers and `@earendil-works/pi-codemode` runs model-written JavaScript that calls tools. [examples/mcp-codemode](examples/mcp-codemode) wraps both as `AgentTool`s: one tool per MCP tool, and a `codemode` tool whose scripts call the agent's tools through `runToolCall()`, so `beforeToolCall` and `afterToolCall` apply to those calls too.
## Proxy Usage
For browser apps that proxy through a backend:
@@ -484,8 +532,7 @@ For direct control without the Agent class:
import { agentLoop, agentLoopContinue } from "@earendil-works/pi-agent-core";
const context: AgentContext = {
systemPrompt: "You are helpful.",
messages: [],
messages: [{ role: "system", content: "You are helpful.", timestamp: Date.now() }],
tools: [],
};

34
third_party/pi/packages/agent/benchmark/session/README.md vendored
View File

@@ -1,34 +0,0 @@
# Session benchmarks
Run these commands from `packages/agent`.
These benchmarks use deterministic synthetic data and make no claim to reproduce production traffic. Storage and repository fork datasets are linear branches of user-message entries with fixed 256-byte text payloads, deterministic IDs, and 250-entry seed transactions. Repository catalog datasets contain deterministic closed sessions.
## Timing
The timing suite runs scenarios against every target in `storage-targets.ts` and `session-repo-targets.ts`. Read scenarios reuse one immutable fixture per backend/dataset and validate their result once before timing. Every warmup and measured write runs once against an independently prepared fixture, so entry counts, sequence numbers, and durable artifacts start in equivalent states. Implementations are registered under the same Vitest suite so additional backends are directly comparable.
Storage writes cover a single message, a 100-message transaction, and a mixed message/register/usage append to a 1k-entry synthetic branch. Repository scenarios cover creating an empty session, opening or deleting a closed empty session, listing 100, 1k, and 10k closed sessions, and forking the current branch of open 1k- and 10k-entry source sessions. Fixture preparation, transaction generation, and validation happen outside the measured callback.
```sh
npm run bench:session:timing
npm run bench:session:timing -- -t "scan latest"
```
## Loaded footprint
The memory profile starts a fresh Node.js process with forced GC for every backend/dataset pair. Its baseline is recorded before generating and ingesting the dataset, so the result includes retained payload data and backend structures. Temporary generation data should be collected before the final reading, while RSS can still reflect allocator growth.
```sh
npm run bench:session:memory
```
## Storage allocation sampling
The allocation profile prebuilds all transactions, then samples allocations made while committing them. This excludes synthetic payload generation and reports estimated cumulative allocated bytes, including objects later collected, plus the largest sampled allocation sites. V8 sampling is approximate and does not report an exact object count.
```sh
npm run bench:session:allocations
```
Timing results are not CI performance gates. Run them on an otherwise idle machine and compare results produced on the same hardware and Node.js version.

57
third_party/pi/packages/agent/benchmark/session/allocation-profile-worker.ts vendored
View File

@@ -1,57 +0,0 @@
import type { HeapProfiler } from "node:inspector";
import { Session } from "node:inspector/promises";
import {
STORAGE_BENCHMARK_DATASETS,
generateStorageBenchmarkSeedTransactions,
} from "../../src/harness/session/testing/index.ts";
import { storageBenchmarkTargets } from "./storage-targets.ts";
const SAMPLING_INTERVAL_BYTES = 4_096;
const TOP_SITE_COUNT = 5;
function collectAllocationSites(node: HeapProfiler.SamplingHeapProfileNode, sites: Map<string, number>): number {
const { functionName, url, lineNumber, columnNumber } = node.callFrame;
const site = `${functionName || "(anonymous)"} (${url || "native"}:${lineNumber + 1}:${columnNumber + 1})`;
if (node.selfSize > 0) sites.set(site, (sites.get(site) ?? 0) + node.selfSize);
let total = node.selfSize;
for (const child of node.children) total += collectAllocationSites(child, sites);
return total;
}
const [targetName, datasetName] = process.argv.slice(2);
const target = storageBenchmarkTargets.find((candidate) => candidate.name === targetName);
if (target === undefined) throw new Error(`Unknown storage benchmark target: ${targetName}`);
const dataset = STORAGE_BENCHMARK_DATASETS.find((candidate) => candidate.name === datasetName);
if (dataset === undefined) throw new Error(`Unknown storage benchmark dataset: ${datasetName}`);
// Inputs and the empty fixture exist before sampling so the profile measures
// allocation caused by Storage.commit(), not synthetic payload generation.
const transactions = [...generateStorageBenchmarkSeedTransactions(dataset)];
await using fixture = await target.createFixture();
const inspector = new Session();
inspector.connect();
try {
await inspector.post("HeapProfiler.startSampling", {
samplingInterval: SAMPLING_INTERVAL_BYTES,
includeObjectsCollectedByMajorGC: true,
includeObjectsCollectedByMinorGC: true,
});
for (const transaction of transactions) await fixture.storage.commit(transaction);
const { profile } = await inspector.post("HeapProfiler.stopSampling");
const sites = new Map<string, number>();
const allocatedBytes = collectAllocationSites(profile.head, sites);
console.log(
JSON.stringify({
target: target.name,
dataset: dataset.name,
entryCount: dataset.entryCount,
allocatedBytes,
topSites: [...sites.entries()]
.map(([site, bytes]) => ({ site, bytes }))
.sort((left, right) => right.bytes - left.bytes)
.slice(0, TOP_SITE_COUNT),
}),
);
} finally {
inspector.disconnect();
}

55
third_party/pi/packages/agent/benchmark/session/allocation-profile.ts vendored
View File

@@ -1,55 +0,0 @@
import { execFile } from "node:child_process";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { STORAGE_BENCHMARK_DATASETS } from "../../src/harness/session/testing/index.ts";
import { storageBenchmarkTargets } from "./storage-targets.ts";
interface AllocationSiteResult {
readonly site: string;
readonly bytes: number;
}
interface AllocationProfileResult {
readonly target: string;
readonly dataset: string;
readonly entryCount: number;
readonly allocatedBytes: number;
readonly topSites: readonly AllocationSiteResult[];
}
const execFileAsync = promisify(execFile);
const packageRoot = fileURLToPath(new URL("../..", import.meta.url));
const workerPath = fileURLToPath(new URL("./allocation-profile-worker.ts", import.meta.url));
const tsconfigPath = fileURLToPath(new URL("../tsconfig.json", import.meta.url));
const results: AllocationProfileResult[] = [];
for (const target of storageBenchmarkTargets) {
for (const dataset of STORAGE_BENCHMARK_DATASETS) {
const { stdout } = await execFileAsync(
process.execPath,
["--import", "tsx", workerPath, target.name, dataset.name],
{ cwd: packageRoot, env: { ...process.env, TSX_TSCONFIG_PATH: tsconfigPath } },
);
results.push(JSON.parse(stdout) as AllocationProfileResult);
}
}
const mebibytes = (bytes: number): string => (bytes / 1024 / 1024).toFixed(2);
console.log("Storage allocation sampling (inputs generated before sampling):");
console.table(
results.map((result) => ({
backend: result.target,
dataset: result.dataset,
"allocated MiB": mebibytes(result.allocatedBytes),
"allocated bytes/entry": Math.round(result.allocatedBytes / result.entryCount),
})),
);
for (const result of results) {
console.log(`\n${result.target}, ${result.dataset}, top sampled allocation sites:`);
console.table(
result.topSites.map(({ site, bytes }) => ({
site,
"sampled MiB": mebibytes(bytes),
})),
);
}

179
third_party/pi/packages/agent/benchmark/session/benchmark.ts vendored
View File

@@ -1,179 +0,0 @@
import { strictEqual } from "node:assert/strict";
import { bench, describe } from "vitest";
export interface BenchmarkTarget<TFixture extends AsyncDisposable> {
readonly name: string;
createFixture(): Promise<TFixture>;
}
const READ_BENCHMARK_OPTIONS = {
time: 500,
iterations: 10,
warmupTime: 100,
warmupIterations: 5,
} as const;
const WRITE_BENCHMARK_ITERATIONS = 30;
const WRITE_BENCHMARK_WARMUP_ITERATIONS = 5;
const WRITE_BENCHMARK_OPTIONS = {
time: 0,
iterations: WRITE_BENCHMARK_ITERATIONS,
warmupTime: 0,
warmupIterations: WRITE_BENCHMARK_WARMUP_ITERATIONS,
} as const;
interface NamedBenchmarkDataset {
readonly name: string;
}
interface ReadBenchmarkScenario<TSubject, TDataset> {
readonly name: string;
expectedResult(dataset: TDataset): number;
run(subject: TSubject, dataset: TDataset): Promise<number>;
}
interface RegisterReadBenchmarksOptions<
TDataset extends NamedBenchmarkDataset,
TFixture extends AsyncDisposable,
TSubject,
> {
readonly datasets: readonly TDataset[];
readonly targets: readonly BenchmarkTarget<TFixture>[];
readonly scenarios: readonly ReadBenchmarkScenario<TSubject, TDataset>[];
prepare(fixture: TFixture, dataset: TDataset): Promise<void>;
getSubject(fixture: TFixture): TSubject;
}
interface PreparedReadFixture<TFixture> {
readonly datasetName: string;
readonly targetName: string;
readonly fixture: TFixture;
}
interface NamedBenchmarkScenario {
readonly name: string;
}
interface RegisterWriteBenchmarksOptions<
TScenario extends NamedBenchmarkScenario,
TFixture extends AsyncDisposable,
TSubject,
> {
readonly targets: readonly BenchmarkTarget<TFixture>[];
readonly scenarios: readonly TScenario[];
prepare(fixture: TFixture, scenario: TScenario): Promise<TSubject>;
expectedResult(scenario: TScenario): number;
run(subject: TSubject, scenario: TScenario): Promise<number>;
}
interface PreparedWriteFixtures<TSubject> {
readonly scenarioName: string;
readonly targetName: string;
readonly pendingSubjects: TSubject[];
}
/** Prepares, validates, registers, and disposes one immutable fixture per target and dataset. */
export async function registerReadBenchmarks<
TDataset extends NamedBenchmarkDataset,
TFixture extends AsyncDisposable,
TSubject,
>(options: RegisterReadBenchmarksOptions<TDataset, TFixture, TSubject>): Promise<void> {
const fixtures: TFixture[] = [];
const preparedFixtures: PreparedReadFixture<TFixture>[] = [];
process.once("beforeExit", async () => {
await Promise.all(fixtures.map((fixture) => fixture[Symbol.asyncDispose]()));
});
for (const dataset of options.datasets) {
for (const target of options.targets) {
const fixture = await target.createFixture();
fixtures.push(fixture);
await options.prepare(fixture, dataset);
const subject = options.getSubject(fixture);
for (const scenario of options.scenarios) {
strictEqual(await scenario.run(subject, dataset), scenario.expectedResult(dataset));
}
preparedFixtures.push({ datasetName: dataset.name, targetName: target.name, fixture });
}
}
for (const dataset of options.datasets) {
for (const scenario of options.scenarios) {
describe(`${scenario.name} (${dataset.name})`, () => {
for (const target of options.targets) {
const prepared = preparedFixtures.find(
(candidate) => candidate.datasetName === dataset.name && candidate.targetName === target.name,
);
if (prepared === undefined) throw new Error("Benchmark fixture was not initialized");
const subject = options.getSubject(prepared.fixture);
bench(
target.name,
async () => {
await scenario.run(subject, dataset);
},
READ_BENCHMARK_OPTIONS,
);
}
});
}
}
}
/** Prepares, validates, registers, and disposes one equivalent fixture per write invocation. */
export async function registerWriteBenchmarks<
TScenario extends NamedBenchmarkScenario,
TFixture extends AsyncDisposable,
TSubject,
>(options: RegisterWriteBenchmarksOptions<TScenario, TFixture, TSubject>): Promise<void> {
const fixtures: TFixture[] = [];
const preparedFixtures: PreparedWriteFixtures<TSubject>[] = [];
process.once("beforeExit", async () => {
await Promise.all(fixtures.map((fixture) => fixture[Symbol.asyncDispose]()));
});
for (const scenario of options.scenarios) {
for (const target of options.targets) {
const validationFixture = await target.createFixture();
fixtures.push(validationFixture);
const validationSubject = await options.prepare(validationFixture, scenario);
strictEqual(await options.run(validationSubject, scenario), options.expectedResult(scenario));
const pendingSubjects: TSubject[] = [];
for (
let index = 0;
index < WRITE_BENCHMARK_WARMUP_ITERATIONS + WRITE_BENCHMARK_ITERATIONS;
index++
) {
const fixture = await target.createFixture();
fixtures.push(fixture);
pendingSubjects.push(await options.prepare(fixture, scenario));
}
preparedFixtures.push({ scenarioName: scenario.name, targetName: target.name, pendingSubjects });
}
}
for (const scenario of options.scenarios) {
describe(scenario.name, () => {
for (const target of options.targets) {
const prepared = preparedFixtures.find(
(candidate) => candidate.scenarioName === scenario.name && candidate.targetName === target.name,
);
if (prepared === undefined) throw new Error("Benchmark fixtures were not initialized");
bench(
target.name,
async () => {
const subject = prepared.pendingSubjects.shift();
if (subject === undefined) throw new Error("Benchmark fixture pool was exhausted");
await options.run(subject, scenario);
},
WRITE_BENCHMARK_OPTIONS,
);
}
});
}
}

34
third_party/pi/packages/agent/benchmark/session/loaded-footprint-worker.ts vendored
View File

@@ -1,34 +0,0 @@
import {
STORAGE_BENCHMARK_DATASETS,
seedStorageBenchmark,
} from "../../src/harness/session/testing/index.ts";
import { storageBenchmarkTargets } from "./storage-targets.ts";
function collectGarbage(): void {
if (globalThis.gc === undefined) throw new Error("Loaded footprint measurement requires Node.js --expose-gc");
for (let index = 0; index < 3; index++) globalThis.gc();
}
const [targetName, datasetName] = process.argv.slice(2);
const target = storageBenchmarkTargets.find((candidate) => candidate.name === targetName);
if (target === undefined) throw new Error(`Unknown storage benchmark target: ${targetName}`);
const dataset = STORAGE_BENCHMARK_DATASETS.find((candidate) => candidate.name === datasetName);
if (dataset === undefined) throw new Error(`Unknown storage benchmark dataset: ${datasetName}`);
await using fixture = await target.createFixture();
collectGarbage();
const before = process.memoryUsage();
await seedStorageBenchmark(fixture.storage, dataset);
collectGarbage();
const after = process.memoryUsage();
console.log(
JSON.stringify({
target: target.name,
dataset: dataset.name,
entryCount: dataset.entryCount,
heapUsedBytes: after.heapUsed - before.heapUsed,
rssBytes: after.rss - before.rss,
externalBytes: after.external - before.external,
}),
);

51
third_party/pi/packages/agent/benchmark/session/loaded-footprint.ts vendored
View File

@@ -1,51 +0,0 @@
import { execFile } from "node:child_process";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { STORAGE_BENCHMARK_DATASETS } from "../../src/harness/session/testing/index.ts";
import { storageBenchmarkTargets } from "./storage-targets.ts";
interface LoadedFootprintResult {
readonly target: string;
readonly dataset: string;
readonly entryCount: number;
readonly heapUsedBytes: number;
readonly rssBytes: number;
readonly externalBytes: number;
}
const execFileAsync = promisify(execFile);
const packageRoot = fileURLToPath(new URL("../..", import.meta.url));
const workerPath = fileURLToPath(new URL("./loaded-footprint-worker.ts", import.meta.url));
const tsconfigPath = fileURLToPath(new URL("../tsconfig.json", import.meta.url));
const results: LoadedFootprintResult[] = [];
for (const target of storageBenchmarkTargets) {
for (const dataset of STORAGE_BENCHMARK_DATASETS) {
const { stdout } = await execFileAsync(
process.execPath,
[
"--expose-gc",
"--import",
"tsx",
workerPath,
target.name,
dataset.name,
],
{ cwd: packageRoot, env: { ...process.env, TSX_TSCONFIG_PATH: tsconfigPath } },
);
results.push(JSON.parse(stdout) as LoadedFootprintResult);
}
}
const mebibytes = (bytes: number): string => (bytes / 1024 / 1024).toFixed(2);
console.log("Loaded footprint (synthetic payloads generated and ingested after the baseline):");
console.table(
results.map((result) => ({
backend: result.target,
dataset: result.dataset,
"heap MiB": mebibytes(result.heapUsedBytes),
"heap bytes/entry": Math.round(result.heapUsedBytes / result.entryCount),
"RSS MiB": mebibytes(result.rssBytes),
"external MiB": mebibytes(result.externalBytes),
})),
);

22
third_party/pi/packages/agent/benchmark/session/session-repo-targets.ts vendored
View File

@@ -1,22 +0,0 @@
import { MemorySessionRepo } from "../../src/harness/session/index.ts";
import type { SessionRepo } from "../../src/harness/session/types.ts";
import type { BenchmarkTarget } from "./benchmark.ts";
export interface SessionRepoBenchmarkFixture extends AsyncDisposable {
readonly repo: SessionRepo;
}
const NOW = 1_700_000_000_000;
export const sessionRepoBenchmarkTargets = [
{
name: "memory",
createFixture() {
const repo = new MemorySessionRepo({ now: () => NOW });
return Promise.resolve({
repo,
[Symbol.asyncDispose]: () => repo.close(),
});
},
},
] satisfies readonly BenchmarkTarget<SessionRepoBenchmarkFixture>[];

62
third_party/pi/packages/agent/benchmark/session/session-repo.bench.ts vendored
View File

@@ -1,62 +0,0 @@
import {
SESSION_REPO_CATALOG_BENCHMARK_DATASETS,
SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS,
SESSION_REPO_CATALOG_WRITE_BENCHMARK_SCENARIOS,
SESSION_REPO_FORK_BENCHMARK_DATASETS,
SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS,
seedSessionRepoCatalogBenchmark,
seedSessionRepoForkBenchmark,
} from "../../src/harness/session/testing/index.ts";
import { registerReadBenchmarks, registerWriteBenchmarks } from "./benchmark.ts";
import { sessionRepoBenchmarkTargets } from "./session-repo-targets.ts";
await registerReadBenchmarks({
datasets: SESSION_REPO_CATALOG_BENCHMARK_DATASETS,
targets: sessionRepoBenchmarkTargets,
scenarios: SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS,
async prepare(fixture, dataset) {
await seedSessionRepoCatalogBenchmark(fixture.repo, dataset);
},
getSubject(fixture) {
return fixture.repo;
},
});
await registerWriteBenchmarks({
targets: sessionRepoBenchmarkTargets,
scenarios: SESSION_REPO_CATALOG_WRITE_BENCHMARK_SCENARIOS,
prepare(fixture, scenario) {
return scenario.prepare(fixture.repo);
},
expectedResult(scenario) {
return scenario.expectedResult;
},
run(operation) {
return operation.run();
},
});
const forkBenchmarks = SESSION_REPO_FORK_BENCHMARK_DATASETS.flatMap((dataset) =>
SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS.map((scenario) => ({
name: `${scenario.name} (${dataset.name})`,
dataset,
scenario,
})),
);
await registerWriteBenchmarks({
targets: sessionRepoBenchmarkTargets,
scenarios: forkBenchmarks,
async prepare(fixture, benchmark) {
return {
repo: fixture.repo,
source: await seedSessionRepoForkBenchmark(fixture.repo, benchmark.dataset),
};
},
expectedResult(benchmark) {
return benchmark.scenario.expectedResult(benchmark.dataset);
},
run(subject, benchmark) {
return benchmark.scenario.run(subject.repo, subject.source, benchmark.dataset);
},
});

18
third_party/pi/packages/agent/benchmark/session/storage-targets.ts vendored
View File

@@ -1,18 +0,0 @@
import { MemoryStorage } from "../../src/harness/session/memory.ts";
import type { StorageFixture } from "../../src/harness/session/testing/index.ts";
import type { BenchmarkTarget } from "./benchmark.ts";
const NOW = 1_700_000_000_000;
export const storageBenchmarkTargets = [
{
name: "memory",
createFixture() {
const storage = new MemoryStorage({ now: () => NOW });
return Promise.resolve({
storage,
[Symbol.asyncDispose]: () => storage.close(),
});
},
},
] satisfies readonly BenchmarkTarget<StorageFixture>[];

35
third_party/pi/packages/agent/benchmark/session/storage.bench.ts vendored
View File

@@ -1,35 +0,0 @@
import {
STORAGE_BENCHMARK_DATASETS,
STORAGE_READ_BENCHMARK_SCENARIOS,
STORAGE_WRITE_BENCHMARK_SCENARIOS,
seedStorageBenchmark,
} from "../../src/harness/session/testing/index.ts";
import { registerReadBenchmarks, registerWriteBenchmarks } from "./benchmark.ts";
import { storageBenchmarkTargets } from "./storage-targets.ts";
await registerReadBenchmarks({
datasets: STORAGE_BENCHMARK_DATASETS,
targets: storageBenchmarkTargets,
scenarios: STORAGE_READ_BENCHMARK_SCENARIOS,
prepare(fixture, dataset) {
return seedStorageBenchmark(fixture.storage, dataset);
},
getSubject(fixture) {
return fixture.storage;
},
});
await registerWriteBenchmarks({
targets: storageBenchmarkTargets,
scenarios: STORAGE_WRITE_BENCHMARK_SCENARIOS,
async prepare(fixture, scenario) {
await scenario.prepare?.(fixture.storage);
return fixture.storage;
},
expectedResult(scenario) {
return scenario.writeCount;
},
run(storage, scenario) {
return scenario.run(storage);
},
});

12
third_party/pi/packages/agent/benchmark/tsconfig.json vendored
View File

@@ -1,12 +0,0 @@
{
"extends": "../../../tsconfig.base.json",
"compilerOptions": {
"baseUrl": "../../..",
"noEmit": true,
"paths": {
"@earendil-works/pi-ai": ["packages/ai/src/index.ts"],
"@earendil-works/pi-ai/*": ["packages/ai/src/*"]
}
},
"include": ["./**/*.ts", "../src/**/*.ts"]
}

355
third_party/pi/packages/agent/docs/assistant-durability.md vendored
View File

@@ -1,355 +0,0 @@
# Assistant partial durability — implementation handoff
This document specifies durable partial assistant messages for ordinary assistant generation and deferred-response polling. It builds on:
- bound typed value/list addresses from `values.md`;
- `AssistantMessageFrame`, `AssistantMessageFrameEncoder`, and `reduceAssistantMessageFrames()` from `@earendil-works/pi-ai`;
- the assistant intent/effect/settlement state machine in `harness.md`.
The design persists compact replayable stream frames without making them operation-state authority and without storing a growing full partial message on every update.
## Goals
1. Reconstruct the latest committed partial assistant message after process loss.
2. Preserve provider stream order without repeated full-snapshot write amplification.
3. Avoid provider backpressure from storage commits.
4. Reuse pi-ai's canonical frame conversion and reduction semantics.
5. Preserve current public assistant event ordering.
6. Keep operation `effect_pending` state authoritative for recovery.
7. Delete all partial frames atomically with normal or synthetic response settlement.
## Non-goals
- Inferring provider completion from frames.
- Persisting terminal `done`/`error` events separately from response settlement.
- Exactly-once provider requests.
- Public frame cursors or a frame-persistence event.
- Generic batching, timers, coalescing, or flush APIs.
- Persisting structural summary-generation streams.
- Persisting arbitrary provider SDK events or repeated full `partial` snapshots.
## Storage
Built-in address constructor in `session/values.ts`:
```ts
export const pendingAssistantFrames = (
operationId: string,
responseEntryId: string,
) => list<AssistantMessageFrame>(
"pi.pending.assistant_frame",
`${operationId}:${responseEntryId}`,
);
```
The procedure binds one exact address for ordinary generation or a deferred poll:
```ts
const frames = pendingAssistantFrames(operationId, responseEntryId);
```
`responseEntryId` is already reserved in assistant/deferred `effect_pending` state. No frame count, cursor, or list identity is stored in operation state, and later list operations receive only `frames`—never another key.
Each list element is one `AssistantMessageFrame`. The storage transaction's global write sequence orders frames. The list is auxiliary:
- missing is valid;
- it does not prove request admission, completion, success, or failure;
- it never selects the restart point;
- base restore does not read it.
## Frame contract
Create one pi-ai encoder per provider stream and feed it every event in order:
```ts
const encoder = new AssistantMessageFrameEncoder();
const frame = encoder.encode(event);
```
`partial` is the provider's shared live response-so-far helper, not an event-time snapshot. The encoder keeps per-open-block counters and trims text/thinking delta prefixes already represented by an advanced block-start snapshot. It temporarily buffers only the raw JSON prefix needed to synchronize an already-advanced tool call, then emits one checkpoint and resumes compact deltas. It never clones the growing full message on every event.
- `start` produces an empty-content metadata frame;
- a non-terminal event produces zero or one frame;
- covered queued deltas produce no frame;
- terminal `done` and `error` produce no frame because final response settlement is separate;
- a setup `error` before `start` is valid and produces no frames;
- text/thinking/tool end frames contain the authoritative completed block value;
- completed tool-call arguments remain unvalidated against a tool schema.
Do not define a second harness frame codec or reducer. Persistence stores the exported pi-ai value directly; hydration calls `reduceAssistantMessageFrames()`.
Provider event blocks may interleave. Encoding and reduction rely on `contentIndex`, never block contiguity. Text and ordinary thinking blocks must be empty when `*_start` is published and then append only through matching deltas until end; redacted thinking may be complete at start and emit no deltas. Streaming tool calls start with empty arguments and emit their complete raw JSON through deltas; a provider that starts with complete arguments must emit a cumulative delta prefix that parses to that snapshot at an event boundary before later argument deltas.
## Fresh stream scheduling
The simplest scheduling is deliberate: one list-append transaction per converted frame, enqueued without awaiting storage in the provider loop.
For every `start` or update event:
```text
encode the event against the per-stream frame encoder
→ when a frame is returned, synchronously enqueue invocation-fenced appendList(frames, frame)
→ attach the ordinary harness-fault observer to the returned promise
→ replace process-local latestFrameWrite promise reference
→ emit and await the existing message_start/message_update event
→ consume the next provider event
```
`appendList()` is called synchronously for each returned frame, so all frame mutations enter the Session line in provider-event order. The procedure does not await each write. Every promise is immediately observed for fault propagation before the latest reference replaces its predecessor. Covered queued events allocate no durable frame or write. Encoder state is proportional to open block count plus the unsynchronized prefix of an active tool-call JSON stream; it retains no second full assistant message. Model output limits bound queued frame bytes to bounded output plus frame/transaction overhead.
The event side retains existing behavior: `AssistantStreamObserver.start/update` awaits `events.emit()` for every event. There is no separate latest event-delivery promise because no assistant event delivery remains outstanding when the provider loop advances.
When the provider stream settles:
```text
stop frame admission
→ await latestFrameWrite when present
→ run after_response
→ emit message_end
→ classify and commit final response settlement
```
The Session mutation line is FIFO, so completion of `latestFrameWrite` implies every earlier append completed. There is no array of promises, timer, batcher, active/waiting state, coalescer, or public/internal flush method.
A failed frame append faults the harness before `after_response` starts. The complete final response remains process-local and does not commit after a storage fault.
## Normal settlement
Every assistant response settlement that used a frame list deletes that exact list in the same transaction as the immutable response entry, usage, tip, and next operation state:
```text
TX[
insert response entry R,
insert usage U,
setValue(branchTip(lane), R),
deleteList(frames),
setValue(operationState(operationId), classified next state)
]
```
This applies to:
- successful assistant responses;
- provider `error`/`aborted` responses;
- valid deferred responses;
- deferred poll responses.
`after_response` may transform the final response before settlement. Frames preserve provider-stream observation, while the immutable entry remains the post-hook canonical result.
A crash after the final frame append but before settlement still restores `effect_pending`; frames do not turn a complete-looking draft into a settled response.
## Unknown-outcome generation recovery
An orphaned assistant generation `effect_pending` has no surviving provider stream. Activation:
1. constructs `frames = pendingAssistantFrames(operationId, responseEntryId)` from current typed state and reads bounded pages from that exact address;
2. reduces frame values with `reduceAssistantMessageFrames()`;
3. constructs a harness-owned synthetic assistant response under the already-reserved response ID;
4. preserves reconstructed partial content and safe message identity metadata when frames exist;
5. sets `stopReason: "error"`, zero usage, and an explicit interruption/unknown-outcome `errorMessage`/diagnostic;
6. commits the synthetic response, zero-usage row, frame-list deletion, tip, and ordinary retry/failure state atomically.
Required warning meaning:
```text
The provider request was interrupted. The preceding content is the latest
committed partial response; newer live output may be missing, and the external
request outcome is unknown.
```
If no start frame committed, the harness constructs the same synthetic error with empty content from the captured model/API identity.
The synthetic response follows ordinary assistant error classification:
- attempts remain → insert the error response and enter the ordinary retry-wait/next-attempt path;
- cap reached → insert it and terminal-fail the operation in the same settlement transaction.
Error responses remain durable transcript history but are omitted from later provider context by the existing projection rule. Partial tool calls inside the interrupted error response never execute.
Recovery does not run `after_response`: there is no trustworthy complete provider result to transform.
## Cancellation
A live cancelled provider stream settles through its ordinary final `aborted` response. All accepted frames are awaited first, and normal settlement deletes the list.
For restored cancelled assistant/deferred `effect_pending`, cancellation reconciliation:
- reduces available frames;
- constructs a synthetic `aborted` response preserving the committed partial content;
- uses zero usage and the existing reserved IDs;
- atomically inserts the response and deletes the frame list;
- starts no provider request and runs no response hook.
Cancellation wins classification even if reduced content appears complete.
## Deferred polling
A deferred poll that returns an `AssistantMessageEventStream` uses the same `pendingAssistantFrames(operationId, responseEntryId)` address constructor.
- normal pending/ready/error settlement deletes the poll's frame list;
- restored cancellation synthesizes an aborted response from frames;
- an unknown restored poll without a permit remains suspended and may expose its durable partial in snapshots;
- when a poll permit replaces the unknown poll with fresh reserved response/usage IDs, that intent transaction deletes the abandoned old frame-list address;
- the replacement poll starts a fresh list under its fresh response ID.
Frames never change poll-number rules.
## Structural generation scope
Structural summary-generation streams remain process-local. They do not emit public assistant-message lifecycle and may span multiple nested provider requests before one structural publication. Existing attempt-level retry and usage recovery remains authoritative.
Do not store their intermediate text at a `pendingAssistantFrames(...)` address in this slice. If structural partial diagnostics become a requirement, add a separate explicitly scoped consumer rather than silently reusing transcript-assistant semantics.
## Snapshots and reconnect
`LaneSnapshot.streamingMessage` means the latest observed partial assistant message, not proof that a provider stream is currently attached.
Precedence:
1. newest process-local partial while a live stream is owned;
2. otherwise, for assistant/deferred `effect_pending`, the value reduced from committed frame pages;
3. otherwise absent.
A restored lane may therefore be `suspended` with a non-undefined `streamingMessage`. The field remains outside `transcript`; only `entry_added` moves a complete response into transcript history and clears the partial.
Snapshot hydration constructs the exact bound frame address from trusted typed operation state and enforces the assistant consumer's total frame/page budget. It does not scan arbitrary lists or perform a broad semantic restore audit.
Reconnect replays no historical `message_start` or `message_update` events. The snapshot carries the durable partial. Recovery later emits the ordinary recovery-tagged synthetic message lifecycle for the response it actually settles.
## Events
A started generation retains this live event ordering:
```text
message_start
→ message_update* each listener delivery awaited by provider loop
→ await latest frame write
→ after_response
→ message_end
→ atomic response settlement + frame-list delete
→ entry_added
→ usage
```
A request setup failure may produce `error` before `start`; that path emits no `message_start` or frame and proceeds through `after_response`, `message_end`, and ordinary error settlement. Successful `done` and updates before `start` are protocol defects.
Frame commits emit ordinary storage telemetry only. There is no public frame event and no claim that a `message_update` was durable. `entry_added` remains the only proof that the final assistant entry committed.
A crash may occur after a live update event but before its asynchronously queued frame append commits. Reconnect then shows the latest committed frame prefix, which may be older than the last live event.
## Close, fault, and external finalization
Close is a controlled crash:
- no synthetic response is written;
- already-enqueued frame commits are ordinary admitted session work and may finish under the close barrier;
- process loss may discard mutations that had not committed;
- reopen restores the latest committed frame prefix under unchanged `effect_pending` state.
A frame-commit storage failure faults the harness. No later response settlement commits in that process.
Authorized external finalization deletes the operation-owned frame-list address in its terminal transaction. Every append mutation verifies current operation/response ownership on the Session line:
- append first → external terminal cleanup deletes the list;
- finalization first → stale append rejects without recreating state.
## Terminal cleanup, forks, and migrations
Normal/synthetic response settlement should already delete its exact frame address. The operation terminal transaction also defensively constructs and deletes the current operation-owned frame address when state is assistant/deferred `effect_pending`.
Idle forks never copy lists in the `pi.pending.assistant_frame` address family. Precise rewrites and migrations page frame lists and preserve element sequences when retaining them.
A migration changing `AssistantMessageFrame` shape must map every surviving element or explicitly delete the whole list and leave `effect_pending` recovery with no partial. It must never infer completion from legacy frames.
JSONL retains deleted frame bytes until snapshot compaction. Logical deletion is immediate.
## Races
| Race | Required result |
|---|---|
| frame append vs next frame | synchronous lane enqueue preserves provider-event order |
| frame append vs stream settlement | settlement awaits the latest promise; all accepted appends finish first |
| live update event vs frame commit | either may finish first; event is observation and reconnect uses only committed frames |
| append vs external finalization | append first is deleted by cleanup; finalization first fences the append |
| process loss with queued writes | only the committed prefix restores |
| final frame vs response settlement | frame commits first; settlement atomically deletes the list and inserts final entry |
| activation vs snapshot | both reduce the same committed sequence prefix; activation may then settle and clear it |
| unknown generation vs retry | synthetic partial error commits under old reserved IDs before later attempt starts |
| unknown deferred poll vs replacement | old list is deleted with fresh replacement intent; new response ID gets a new list |
## Invariants
1. Scalar assistant/deferred state is the sole restart authority.
2. One effect-pending response ID constructs exactly one assistant frame-list address.
3. Every stored element is an exported pi-ai `AssistantMessageFrame`.
4. Terminal `done`/`error` events are never stored as frames.
5. Frame order is a subsequence of provider event order; zero-frame covered events do not disturb order.
6. Awaiting the latest frame-write promise at stream settlement implies all accepted appends completed.
7. Frames never establish provider completion or suppress unknown-outcome recovery.
8. Final or synthetic response settlement atomically deletes the exact frame list.
9. A restored partial may appear in `streamingMessage` but never in `transcript` before settlement.
10. Interrupted partial tool calls never produce a tool plan because the synthetic response stops with `error`/`aborted`.
11. Structural generation never writes a `pendingAssistantFrames(...)` list in this slice.
12. Terminal cleanup, external finalization, and idle forks leave no operation-owned assistant frame list.
## Required tests
### Frame integration
- one per-stream encoder handles shared live partials and synchronous event bursts without duplicate content;
- each event appends zero or one frame, with covered queued deltas appending nothing;
- `done`/`error` append nothing, including pre-generation error;
- interleaved content indexes preserve sequence;
- provider loop does not await individual frame writes;
- frame appends enqueue synchronously before the next provider event;
- only the latest promise reference is retained;
- awaiting the latest promise implies all earlier writes completed;
- bounded output bounds queued frame memory;
- storage failure prevents `after_response` and faults the harness.
### Settlement and recovery
- every ordinary response class deletes the frame list atomically;
- crash at every frame/settlement boundary;
- no frames and partial text/thinking/tool-call frames;
- authoritative end-frame content survives;
- interrupted generation commits partial synthetic error then retries or fails at cap;
- interrupted partial tool calls never execute;
- recovery uses zero usage and existing reserved IDs;
- cancellation preserves committed partial content in synthetic aborted response;
- `after_response` never runs for restored synthetic settlement.
### Deferred, snapshots, and lifecycle
- deferred poll frame persistence and normal cleanup;
- unknown poll snapshot without permit;
- replacement intent deletes abandoned poll frames;
- live partial takes precedence over durable reduction;
- reopen exposes reduced `streamingMessage` on suspended effect-pending lane;
- no historical update event replay;
- recovery settlement clears partial on `entry_added`;
- structural generation writes no assistant frame list.
### Storage lifecycle
- Memory/JSONL/SQLite reduce identical frame sequences;
- frame list is absent after normal, synthetic, cancellation, and external terminal transitions;
- idle fork excludes frames;
- JSONL compaction removes deleted frame bytes;
- migration maps or explicitly discards every legacy frame;
- instrumentation records append/delete ordering without frame content in telemetry.
## Implementation map
Expected runtime areas:
- built-in `pendingAssistantFrames(operationId, responseEntryId)` address constructor in `session/values.ts`;
- bound value/list storage implementation from `values.md`;
- assistant execution observer and generation procedure;
- deferred polling procedure;
- activation and cancellation recovery;
- snapshot hydration;
- terminal cleanup, forks, and migrations;
- instrumented writer and backend conformance tests.
Implement bound typed value/list addresses first, then frame enqueue/settlement, then recovery/snapshots. Update `harness.md` with this complete lifecycle before implementing runtime assistant parity.

1468
third_party/pi/packages/agent/docs/harness.md vendored
View File

File diff suppressed because it is too large Load Diff

214
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/FINDINGS.md vendored
View File

@@ -1,214 +0,0 @@
# 01-delta: known defects and measured findings
Everything here was found **after** `delta.md` and the prototype implementation beside it were written. None of it is applied to the code in this directory. Production now lives in `packages/chord/src/delta/index.ts`: flush-time dirty tracking resolves D1, and production remeasurement closes D2 without a new API. See [`append-decision.md`](append-decision.md). Each item below remains the original problem, evidence, and candidate-fix record.
Every number was measured with **`node --experimental-strip-types`**, not `tsx`.
That matters: tsx transpiles through esbuild and inflated the same benchmark by
**2.6x** (183 µs vs 63 µs per write). Do not benchmark this module through a
transpiler.
---
## D1. Interleaved paths defeat coalescing (correctness-adjacent, unbounded memory)
**Severity: high.** This is the one to fix first.
### Problem
`flush()` coalesces only **adjacent** ops. A producer that alternates between two
paths therefore never coalesces anything, and the ops array grows with the number
of writes rather than with the size of the value.
This is not hypothetical. It is exactly what bash does: it updates the output text
and a byte counter on every chunk.
```ts
const next = t.state.content[0].text + chunk;
t.state.content[0].text = next.length > CAP ? next.slice(next.length - CAP) : next;
t.state.truncation.totalBytes += chunk.length; // <- the other path
```
### Evidence
50 KB window, 200-byte chunks, no flush in between:
| held-back writes | ops | bytes |
| --- | --- | --- |
| 1 | 3 | 0.3 KB |
| 100 | 201 | 26 KB |
| **1000** | **2001** | **264 KB** |
Without interleaving the same workload gives 1 op and 51 KB. The window is
bounded at 50 KB; the ops describing it are not.
### Why it matters beyond memory
Any design where the sink **holds back** — rate limiting, backpressure, a slow
consumer — depends on held-back writes costing nothing. They currently cost
everything. See `../../04-tool-output/rate-limiting.md`.
### Candidate fix
Fold at **record** time, keyed by path rather than by adjacency: one slot per
path holding at most two ops (`[s]`, `[d]`, `[t]`, `[a]`, or `[t, a]`), emitted in
first-touch order.
A prototype reached **2 ops / 51 KB** on the same workload. It is not included
here because it was not verified to the standard the rest of this unit meets.
**Four folding rules, each a specific claim about the vocabulary:**
1. A later `s` supersedes an earlier `s` on the same path — but **not** an earlier
`d`, which must survive or the key changes position on reinsertion (§5.1).
2. Consecutive `a` merge. Once the merged append is **strictly longer** than the
value, the window has been sliding and the append is the more expensive
spelling, so collapse to `s`. Strictly: while a string is still *growing*
toward its cap the append equals the value, and collapsing there would resend
the whole string every batch.
3. `t` and `a` on one path **commute** — the truncate drops from the front, the
append adds to the end — so a rolling window's `t,a,t,a,...` folds to one of
each. Without this, pairwise merging never fires at all.
4. `t` or `a` following a pending `s` changes nothing: the `s` already means "this
path becomes its current value".
**The trap that prototype hit.** Slots emit in first-touch order, which loses the
fact that a child write happened *before* a later parent write:
```
script : a={x:1}; a.b=99; a={c:2}
producer: {"a":{"c":2}}
replica : {"a":{"c":2,"b":99}} <- WRONG
```
A set or delete must invalidate every pending slot **below** its path, at record
time. The existing `verify-dead` style test passed 1190 sequences while this was
broken, because its generator only nested one level. Any test for this must
generate genuinely nested paths with parent overwrites interleaved between child
writes.
---
## D2. `overlap()` was 94.5% of prototype tracker time — closed
**Historical severity: high. Production decision:** do not add an explicit append/truncate API. The prototype profile was dominated by a slow `startsWith` path; production uses a flattened-slice comparison and measured 17.8–18.7 µs per 200 KB assistant append flush and 2.43–2.46 µs per 50 KB rolling-window flush locally. See [`append-decision.md`](append-decision.md). The original evidence follows.
### Problem
The tracker infers "you appended and evicted" by searching two 50 KB strings.
Each new value produced by `next.slice(-CAP)` is a V8 `SlicedString` — a lazy
substring — and `indexOf`/`endsWith` force it to **flatten**, copying 50 KB per
call.
### Evidence
CPU profile of a 20 000-write rolling-window loop, node, flush at the end:
```
94.5% overlap
2.7% (anon)
1.2% (garbage collector)
0.5% set
0.3% emit
0.0% fold
```
And the flattening is directly observable:
| `overlap()` on a 50 KB window | µs |
| --- | --- |
| the same two strings reused | 37 |
| **a fresh sliced string each call** | **149** |
An earlier benchmark reported 29 µs because it reused two already-flattened
strings 2000 times — a case that never occurs in the real loop. **Do not benchmark
string operations against a reused fixture.**
### Candidate fix
Do not infer what the producer already knows. `ToolOutput` performs the windowing,
so it can emit the ops directly rather than assigning a new string and having the
tracker search for the difference:
```ts
out.append(chunk); // -> ["t", path, dropped], ["a", path, chunk]
```
Two integers — chars appended, chars evicted — describe everything that happened
between any two observations of a bounded window. A prototype of this shape held
**exactly the window** in memory under every load, with no ops array, no overlap
detection and no folding rules at all.
This does **not** remove `overlap`: it stays for the general case where a producer
assigns a whole new string and the delta genuinely has to be discovered. It
removes it from the hot path.
### Do not
Do not "optimise" `overlap` itself before doing the above. Its algorithm is
already correct and bounded (`maxOverlapScan`); the cost is string flattening,
which no probe strategy avoids.
---
## D3. Nested-path proxy overhead is next, and currently invisible
**Severity: unknown — measure before acting.**
`overlap` dominates so completely that nothing else shows in a profile. But two
benchmarks disagree in a way that points at path depth:
| | µs per write |
| --- | --- |
| top-level path (`{ text }`) | 63 |
| nested path (`{ content: [{ text }] }`) | 244 |
4x for depth alone. Each nested property access wraps a child proxy and builds a
cache key by joining the path, so a read of `content[0].text` allocates on every
access.
**Re-profile only if production delta cost becomes material.** This may be the next bottleneck or it may be
nothing; the current numbers cannot distinguish them because `overlap` swamps both.
---
## D4. Things measured and found NOT worth doing
Recorded so nobody repeats the work. All were under **1.5% combined** in the
profile above.
**A path trie replacing `JSON.stringify(path)` keys.** Genuinely faster in
isolation — 6x on the `LaneSnapshot` shape, 93x at 500 distinct paths — because
stringify cost scales with path depth while a trie walk is one `Map.get` per
segment. But the whole slot layer is 0.5% of runtime. Build it only if a current production profile says so.
**Trie as storage vs trie as index.** If a trie is ever built, it should map path
to a **position in the ops array**, not hold the ops. Holding them loses first-touch
ordering and forces a sort at flush to rebuild it. Measured difference: none
(1.4x / 0.9x / 1.0x). The argument is structural, not performance.
**A `maxDepth` heuristic to skip descendant scanning.** Rejected: the flat map was
losing on the common case too, so the heuristic would have hidden a real cost
rather than removed it.
**Eager string concatenation in the `a`+`a` merge.** Real (it rebuilds a growing
string per write) but worth ~5%. Compare lengths and materialise only when
emitting.
---
## D5. Measurement errors made while producing these numbers
Listed because each one produced a confident wrong conclusion, and the same traps
are still there.
- **Benchmarking through `tsx`** — inflated everything 2.6x.
- **Reusing a flattened string** in a string benchmark — hid the real cost 4x.
- **Repetitive fixtures** (`"x".repeat(n)`) — overlap detection hits its candidate
bound and gives up, so the benchmark measures the fallback path, not the real one.
Use varied text.
- **Measuring a cache hit as if it were work** — `Markdown.render()` returns
`cachedLines` when text and width are unchanged, so a repeat-the-same-call loop
measured 0.065 ms for something that costs 0.86 ms.
- **Reasoning about complexity while the constant dominated** — twice.
- **Measuring heap instead of retained size** — a fixture generating 800 x 1 MB
strings reported 40 MB of "growth" that was its own garbage.

39
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/append-decision.md vendored
View File

@@ -1,39 +0,0 @@
# Decision: no explicit text append/truncate API
**Status: closed.** Do not add `appendText`, explicit truncate, a text-specific dirty-node kind, or proxy-to-tracker lookup machinery.
## Why it was considered
The tracker marks dirty paths and compares them with its accepted baseline at `flush()`. For a growing or rolling string, confirming the relationship can scan the retained string even though the producer already knows it appended or evicted text.
The original benchmark made this look expensive because its append fast path used `after.startsWith(before)`. V8 walked the producer's cons string character by character. Production Chord now uses:
```ts
if (after.length > before.length && after.slice(0, before.length) === before) {
// emit append
}
```
The slice flattens once and comparison uses the native string path. Flush-time dirty tracking also removed the prototype's retained-op amplification.
## Local confirmation
Measured 2026-09-01 against `origin/dev` at `1a7bc80e7`, using `packages/chord/src/delta/index.ts` directly under Node 26.0.0 on an Apple M5 Max. Each workload used 3,000 warmups followed by 11 samples of 10,000 mutation-plus-flush iterations; a second process reproduced the result.
| Workload | Median µs/flush, run 1 | Median µs/flush, run 2 |
| --- | ---: | ---: |
| 200 KB assistant string, append 8 characters | 18.68 | 17.81 |
| 50 KB rolling window, slide 32 varied characters | 2.46 | 2.43 |
| Transcript push, one small entry | 0.74 | 0.78 |
The assistant tracker started with 200,000 varied characters, appended `" abcdef"`, and flushed each append. The rolling tracker started with 50,000 varied characters, then assigned `text.slice(32) + chunk` using a distinct 32-character `chunk:<base36 index>:durable-stream` value on every flush; every flush was asserted to emit `t` + `a`. The transcript tracker pushed `{ id: "e<index>", text: "message <index>" }` and asserted one `p` per flush. Setup and garbage collection were outside each timed loop.
At 100 assistant updates per second, the growing-string case consumes about 1.8 ms/s, approximately 0.18% of one core. The measured rolling-window cost is itself too small to justify the abandoned explicit API.
## Why the API is rejected
The prototype required a text-specific dirty-node state, proxy-to-tracker lookup, repeated-append/drop folding, and interaction rules for later whole-value replacement. It produced subtle silent failures: one implementation fell through to the ordinary differ while tests still passed, and another drifted when drops reached into earlier appends.
That complexity does not justify saving microseconds below the surrounding replication, isolation, and rendering costs. Keep ordinary string mutation and the generic Chord op vocabulary.
Reopen only when a production profile shows delta flush time is a meaningful fraction of a real workload. Re-measure the generic fast path first; a regression there is cheaper and safer to fix than adding a producer-specific API.

826
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta-impl.ts vendored
View File

@@ -1,826 +0,0 @@
// ─────────────────────────────────────────────────────────────────────────────
// harness/delta — intent-recording change tracking over plain JSON.
//
// Depends on nothing else in the harness. Session storage, the runtime and the
// facet host consume it; keep the arrows pointing that way.
// ─────────────────────────────────────────────────────────────────────────────
export type JsonValue = null | boolean | number | string | JsonValue[] | { [k: string]: JsonValue };
export type Seg = string | number;
export type Path = readonly Seg[];
export type NonEmptyPath = readonly [Seg, ...Seg[]];
/** A path inline, or an id assigned by the encoder on second use. */
export type PathRef<P extends Path = Path> = P | number;
/**
* Tuples are the form — in memory, on the wire, on disk.
*
* `r` is the ONLY op that replaces a whole value. `s`/`d`/`a`/`t` cannot target
* the root: the type forbids it. `p` may, and only because a tracked value can
* itself be an array — but a `p` that replaces its entire target is normalised to
* `r`/`s` at record time, so a root `p` is always a partial modification.
*
* `Op` knows nothing about the path dictionary. Interning, id references and
* omitted paths live in `WireOp` and exist only between `encode` and `decode`.
*/
export type Op =
| readonly ["r", JsonValue]
| readonly ["s", NonEmptyPath, JsonValue]
| readonly ["d", NonEmptyPath]
| readonly ["a", NonEmptyPath, string]
| readonly ["t", NonEmptyPath, number]
| readonly ["p", Path, number, number, JsonValue[]];
/**
* What crosses a boundary. Adds two compressions and nothing else:
*
* ["#", id, path] defines an id, emitted on a path's SECOND use
* a numeric PathRef references a previously defined id
* a shortened tuple reuses the previous op's path; arity disambiguates
*
* ["r", value] carries no path, so it encodes to itself — which is why isBase
* works unchanged on either vocabulary.
*/
export type WireOp =
| readonly ["r", JsonValue]
| readonly ["s", PathRef<NonEmptyPath>, JsonValue] | readonly ["s", JsonValue]
| readonly ["d", PathRef<NonEmptyPath>] | readonly ["d"]
| readonly ["a", PathRef<NonEmptyPath>, string] | readonly ["a", string]
| readonly ["t", PathRef<NonEmptyPath>, number] | readonly ["t", number]
| readonly ["p", PathRef, number, number, JsonValue[]]
| readonly ["p", number, number, JsonValue[]]
| readonly ["#", number, Path];
// ─── Classification ──────────────────────────────────────────────────────────
export const isReplace = (op: Op | WireOp): boolean => op[0] === "r";
/**
* A batch begins with a replacement. Flush guarantees `r` is at index 0 or absent,
* so this is exact rather than a heuristic.
*/
export const isBase = (ops: readonly (Op | WireOp)[]): boolean =>
ops.length > 0 && ops[0]![0] === "r";
// ─── Overlap ─────────────────────────────────────────────────────────────────
/**
* Longest suffix of `a` that is a prefix of `b`. Probes with indexOf and verifies
* with endsWith, so the hot loops are native. A hand-written KMP is
* asymptotically equivalent and ~47x slower in practice.
*
* Always correct: the returned n satisfies a.endsWith(b.slice(0, n)).
*/
export function overlap(a: string, b: string, scan: number, probe = 64, maxCandidates = 8): number {
if (a.length === 0 || b.length === 0 || scan === 0) return 0;
const tail = a.length > scan ? a.slice(a.length - scan) : a;
// A probe of length h can only find overlaps of at least h — the head must
// actually occur in `a`. So try a long head first (few candidates, and it
// catches the large overlaps a rolling window produces), then fall back to one
// character, which finds any overlap at the cost of more candidates.
//
// Candidates are bounded because repetitive output — a build log, or any run of
// one character — makes a long head match at thousands of positions, each
// costing a full endsWith. Giving up returns 0, which emits a set: larger, never
// wrong. Unbounded, a 50 KB window of one repeated character took 4.7 s per
// 2000 slides against 90 ms bounded.
for (const h of [Math.min(probe, b.length), 1]) {
const head = b.slice(0, h);
let tried = 0;
for (let k = tail.indexOf(head); k !== -1; k = tail.indexOf(head, k + 1)) {
if (++tried > maxCandidates) break;
const n = tail.length - k;
if (n <= b.length && tail.endsWith(b.slice(0, n))) return n;
}
if (h === 1) break;
}
return 0;
}
// ─── Tracker ─────────────────────────────────────────────────────────────────
export interface TrackerOptions {
maxOverlapScan?: number;
coalesce?: boolean;
}
export interface Tracker<T extends object> {
/**
* Mutate this. Plain TS: assignment, push, splice, delete, all of it.
*
* Assigning to it replaces the whole value: emits `["r", next]` and discards
* ops recorded before it, which describe a value that no longer exists.
*
* `tracker.state = tracker.target` is therefore a checkpoint — it forces a base
* batch without changing anything. Discarding the pending ops is correct
* because the proxy mutates the target directly, so the value already carries
* them.
*/
state: T;
/** The untracked backing object. Reads only — writing here emits nothing. */
readonly target: T;
flush(): Op[];
/**
* Clear pending ops and make the next flush a base batch, without changing
* the value.
*
* Recovery replays from the last base batch, so a producer that wants "at most
* N batches to replay" calls this every N. The pending ops are already
* reflected in the value — the proxy mutates the target directly — so
* discarding them loses nothing.
*/
rebase(): void;
/** Discard without emitting. */
discard(): void;
readonly dirty: boolean;
}
const isObj = (v: unknown): v is object => v !== null && typeof v === "object";
const clone = <T>(v: T): T => (isObj(v) ? (structuredClone(v) as T) : v);
const INDEX = /^(?:0|[1-9]\d*)$/;
const norm = (t: object, k: string | symbol): Seg | symbol =>
typeof k === "symbol" ? k : Array.isArray(t) && INDEX.test(k) ? Number(k) : k;
const MUTATORS = new Set(["push", "pop", "shift", "unshift", "splice", "sort", "reverse", "fill", "copyWithin"]);
export function track<T extends object>(root: T, options: TrackerOptions = {}): Tracker<T> {
const scan = options.maxOverlapScan ?? 65_536;
const doCoalesce = options.coalesce ?? true;
let ops: Op[] = [];
const emit = (op: Op) => {
ops.push(op);
};
/**
* Reject reserved keys at record time, not just on apply.
*
* `JSON.parse('{"__proto__":{}}')` yields an OWN property, so a producer can
* hold state containing one without doing anything unusual. The tracker would
* then emit a path the applier refuses — state that cannot be replicated. Fail
* where the mistake is, at the write.
*/
const guard = (seg: Seg | symbol): Seg => {
if (typeof seg === "symbol") throw new UnsafePathError(String(seg));
if (typeof seg === "string" && FORBIDDEN.has(seg)) throw new UnsafePathError(seg);
return seg;
};
/**
* A splice covering the whole target is a replacement written the long way.
* Only the interception knows the pre-splice length, so this is a record-time
* rewrite, not something flush can recover.
*/
const spliceOrReplace = (path: Path, before: number, index: number, remove: number, items: JsonValue[]) => {
if (index === 0 && remove === before) {
if (path.length === 0) emit(["r", items]);
else emit(["s", path as NonEmptyPath, items]);
return;
}
emit(["p", [...path], index, remove, items]);
};
const cache = new WeakMap<object, Map<string, object>>();
const wrap = <V extends object>(obj: V, path: Path): V => {
let byPath = cache.get(obj);
if (byPath === undefined) cache.set(obj, (byPath = new Map()));
const cacheKey = path.join("\u0000");
const hit = byPath.get(cacheKey);
if (hit !== undefined) return hit as V;
const proxy = new Proxy(obj, {
get(t, k, r) {
if (Array.isArray(t) && typeof k === "string" && MUTATORS.has(k)) {
return (...args: unknown[]) => {
const before = t.length;
const result = (Array.prototype[k as "push"] as Function).apply(t, args);
const items = (xs: unknown[]) => xs.map(clone) as JsonValue[];
switch (k) {
case "push": spliceOrReplace(path, before, before, 0, items(args)); break;
case "unshift": spliceOrReplace(path, before, 0, 0, items(args)); break;
case "pop": if (before > 0) spliceOrReplace(path, before, before - 1, 1, []); break;
case "shift": if (before > 0) spliceOrReplace(path, before, 0, 1, []); break;
case "splice": {
const raw = Number(args[0] ?? 0);
const index = raw < 0 ? Math.max(0, before + raw) : Math.min(raw, before);
const remove = args.length < 2 ? before - index : Math.max(0, Math.min(Number(args[1]), before - index));
spliceOrReplace(path, before, index, remove, items(args.slice(2)));
break;
}
// sort/reverse/fill/copyWithin permute wholesale; intent is not a splice.
default:
if (path.length === 0) emit(["r", clone(t) as JsonValue]);
else emit(["s", path as NonEmptyPath, clone(t) as JsonValue]);
}
return result;
};
}
const v = Reflect.get(t, k, r);
return isObj(v) ? wrap(v, [...path, guard(norm(t, k))]) : v;
},
set(t, k, v, r) {
// `length` is a real mutation but not a document location.
if (Array.isArray(t) && k === "length") {
const before = t.length;
const next = Number(v);
if (next < before) spliceOrReplace(path, before, next, before - next, []);
else if (next > before) emit(["p", [...path], before, 0, Array(next - before).fill(null)]);
return Reflect.set(t, k, v, r);
}
const at = [...path, guard(norm(t, k))] as unknown as NonEmptyPath;
if (v === undefined) { // JSON has no undefined
emit(["d", at]);
return Reflect.deleteProperty(t, k);
}
const prev = (t as Record<string | symbol, unknown>)[k];
if (typeof prev === "string" && typeof v === "string") {
if (prev === v) return true;
if (v.length > prev.length && v.startsWith(prev)) {
emit(["a", at, v.slice(prev.length)]); // fast path, no probe
} else {
const ov = overlap(prev, v, scan);
if (ov === 0) emit(["s", at, v]);
else {
emit(["t", at, prev.length - ov]);
if (v.length > ov) emit(["a", at, v.slice(ov)]);
}
}
} else {
assertJsonValue(v);
emit(["s", at, clone(v) as JsonValue]);
}
return Reflect.set(t, k, v, r);
},
deleteProperty(t, k) {
emit(["d", [...path, guard(norm(t, k))] as unknown as NonEmptyPath]);
return Reflect.deleteProperty(t, k);
},
});
byPath.set(cacheKey, proxy);
return proxy as V;
};
let state = wrap(root, []);
// The first flush is always a base batch. A consumer starts with nothing, so
// a stream that opens with deltas has nothing to apply them to — and that
// fails at runtime, in the consumer, rather than where the mistake is.
let forceBase = true;
return {
get state() { return state; },
// A setter, so the obvious thing works. Without it `tracker.state = next`
// silently swaps the proxy for a plain object and stops tracking.
set state(next: T) {
ops = [];
root = next;
state = wrap(root, []);
forceBase = true;
},
get target() { return root; },
// Same effect as assigning the current value back, minus the re-wrap: the
// value has not changed, so the proxy cache is still valid.
rebase() { ops = []; forceBase = true; },
get dirty() { return ops.length > 0; },
discard() { ops = []; },
flush() {
if (forceBase) {
forceBase = false;
ops = [];
return [["r", clone(root) as JsonValue]];
}
if (ops.length === 0) return [];
let out = ops;
ops = [];
// Everything before a replacement is dead.
const lastReplace = out.findLastIndex(isReplace);
if (lastReplace > 0) out = out.slice(lastReplace);
if (doCoalesce) out = coalesce(dropDead(out));
return out;
},
};
}
const samePath = (a: Op, b: Op): boolean => {
const pa = pathOf(a);
const pb = pathOf(b);
if (pa === undefined || pb === undefined) return false;
return JSON.stringify(pa) === JSON.stringify(pb);
};
function pathOf(op: Op): Path | undefined {
return op[0] === "r" ? undefined : op[1];
}
/**
* Drop ops that a later op makes unreachable.
*
* Ops apply in order, so an op is dead if a later one overwrites the whole
* subtree it lives in. Only `s`, `d` and `r` do that — `a`, `t` and `p` modify
* what is already there, so they never dominate.
*
* O(n * d): walk backwards keeping a map of overwritten paths, and for each op
* test its own prefixes against that map. `d` is the path depth, which is small
* and bounded. The obvious formulation — compare each op against every
* dominator — is O(n^2 * d) and degrades on exactly the wide-flush input this
* pass exists to clean up. Measured: 12-14 microseconds per op, flat from 100 to
* 5000 ops, and 10001 redundant writes collapse to 2 emitted ops.
*
* OBJECT KEY ORDER IS NOT PRESERVED. A producer that does
* `set c; set a; delete c; set c` leaves `c` last, because deleting and
* reinserting moves it. The replica receives only the surviving `set c`, applies
* it to a base where `c` never existed, and `c` lands first. Values are
* identical — verified over 1823 randomised sequences with zero value
* mismatches — but the serialised forms can differ.
*
* This is fine because nothing compares a replica against its producer: the
* consumer holds the state and nobody diffs it back. The one visible case is a
* consumer that iterates keys for display, where two views could order rows
* differently. It requires a delete-and-recreate of the same key within one
* flush. Do not hash or content-address a replicated value.
*/
function dropDead(ops: Op[]): Op[] {
const overwritten = new Map<string, "s" | "d">();
const kept: Op[] = [];
for (let i = ops.length - 1; i >= 0; i--) {
const op = ops[i]!;
if (op[0] === "r") {
kept.push(op);
overwritten.clear();
overwritten.set("", "s"); // the root is now overwritten
continue;
}
const path = opPath(op);
if (path === undefined) { kept.push(op); continue; } // arity form, or an interned id
let dead = overwritten.has("");
let key = "";
for (let k = 0; k < path.length && !dead; k++) {
key += `\u0000${String(path[k])}`;
const kind = overwritten.get(key);
if (kind === undefined) continue;
// A `d` at exactly a later `s`'s path is NOT dead. Dropping it leaves the
// key in its original slot, while the producer's delete-then-set moved it
// to the end: same value, different serialisation.
if (k === path.length - 1 && op[0] === "d" && kind === "s") break;
dead = true;
}
if (dead) continue;
kept.push(op);
if (op[0] === "s" || op[0] === "d") overwritten.set(key, op[0]);
}
return kept.reverse();
}
/** Total, because `Op` paths are always inline. */
function opPath(op: Op): Path | undefined {
return op[0] === "r" ? undefined : op[1];
}
/** Merge adjacent ops describing the same path. Lossless. */
function coalesce(ops: Op[]): Op[] {
const out: Op[] = [];
for (const op of ops) {
const last = out[out.length - 1];
if (last !== undefined && samePath(last, op)) {
if (last[0] === "a" && op[0] === "a") {
out[out.length - 1] = ["a", last[1], `${last[2]}${op[2]}`];
continue;
}
if (last[0] === "s" && op[0] === "s") { // later write wins
out[out.length - 1] = op;
continue;
}
if (last[0] === "s" && op[0] === "a" && typeof last[2] === "string") {
out[out.length - 1] = ["s", last[1], `${last[2]}${op[2]}`];
continue;
}
}
out.push(op);
}
return out;
}
// ─── Path safety ─────────────────────────────────────────────────────────────
/**
* Segments that reach the prototype chain.
*
* `JSON.parse` is safe on its own — it makes `__proto__` an own property. What is
* not safe is `parent[key] = value`, which is exactly what an applier does, and
* paths are data: `["s", ["__proto__", "isAdmin"], true]` pollutes
* `Object.prototype` for the whole process.
*
* Ops arrive from a facet, a plugin compartment, or a tool whose details may echo
* model output, so none of it is trusted input.
*/
const FORBIDDEN = new Set(["__proto__", "constructor", "prototype"]);
export class UnsafePathError extends Error {
// Not a parameter property: Node's --experimental-strip-types rejects those,
// and these files are meant to run under it directly.
readonly segment: Seg;
constructor(segment: Seg) {
super(`unsafe path segment: ${String(segment)}`);
this.segment = segment;
this.name = "UnsafePathError";
}
}
/**
* Rejects anything JSON.stringify would not round-trip.
*
* structuredClone is NOT a JSON check: it clones a Map happily, and the op then
* carries `{}` while the producer keeps a real Map — producer and replica diverge
* silently, forever.
*/
export function assertJsonValue(value: unknown, seen = new Set<unknown>()): void {
if (value === null) return;
const t = typeof value;
if (t === "string" || t === "boolean") return;
if (t === "number") {
if (!Number.isFinite(value as number)) throw new TypeError("non-finite number is not JSON");
return;
}
if (t !== "object") throw new TypeError(`${t} is not a JsonValue`);
if (seen.has(value)) throw new TypeError("cyclic value is not JSON");
seen.add(value);
if (Array.isArray(value)) {
for (const v of value) assertJsonValue(v, seen);
} else {
const proto = Object.getPrototypeOf(value);
if (proto !== Object.prototype && proto !== null) {
throw new TypeError(`${(value as object).constructor?.name ?? "object"} is not a JsonValue`);
}
if (Object.getOwnPropertySymbols(value).length > 0) {
throw new TypeError("symbol keys are not JSON");
}
for (const v of Object.values(value as object)) assertJsonValue(v, seen);
}
seen.delete(value);
}
/**
* Verb, arity and payload shape for a **decoded** op: paths inline, no `#`, no
* short forms. `apply` uses this.
*
* Validating `Op` against the wire grammar would be laxer than the type: a
* two-element `["s", value]` would pass, and `apply` would then read the value as
* a path. Each vocabulary gets the validator that matches it.
*/
export function assertValidOp(op: unknown): void {
if (!Array.isArray(op) || op.length === 0) throw new TypeError("op is not a tuple");
switch (op[0]) {
case "r": if (op.length !== 2) throw new TypeError("r arity"); assertJsonValue(op[1]); return;
case "s": if (op.length !== 3) throw new TypeError("s arity"); assertPathArg(op[1]); assertJsonValue(op[2]); return;
case "d": if (op.length !== 2) throw new TypeError("d arity"); assertPathArg(op[1]); return;
case "a": if (op.length !== 3 || typeof op[2] !== "string") throw new TypeError("a shape"); assertPathArg(op[1]); return;
case "t": if (op.length !== 3 || !Number.isInteger(op[2])) throw new TypeError("t shape"); assertPathArg(op[1]); return;
case "p": {
if (op.length !== 5) throw new TypeError("p arity");
assertPathArg(op[1]);
if (!Number.isInteger(op[2]) || op[2] < 0) throw new TypeError("p index");
if (!Number.isInteger(op[3]) || op[3] < 0) throw new TypeError("p remove");
if (!Array.isArray(op[4])) throw new TypeError("p items");
for (const v of op[4]) assertJsonValue(v);
return;
}
// Silently skipping an unknown verb is how a newer producer's op vanishes.
default: throw new TypeError(`unknown op verb: ${String(op[0])}`);
}
}
function assertPathArg(p: unknown): void {
if (!Array.isArray(p)) throw new TypeError("path is not an array");
assertSafePath(p as Path);
}
/** The same, for the wire grammar: ids and short forms are legal here. */
export function assertValidWireOp(op: unknown): void {
if (!Array.isArray(op) || op.length === 0) throw new TypeError("op is not a tuple");
const [verb] = op as unknown[];
const okRef = (r: unknown): void => {
if (typeof r === "number") {
if (!Number.isInteger(r) || r < 0) throw new TypeError("bad path id");
return;
}
// A string is not a path. Unchecked, `"a".slice(0, -1)` is `""`, so it
// resolves to the ROOT and writes there — a path that is not a path, accepted.
if (!Array.isArray(r)) throw new TypeError("path is not an array");
assertSafePath(r as Path);
};
switch (verb) {
case "r": if (op.length !== 2) throw new TypeError("r arity"); assertJsonValue(op[1]); return;
case "s": if (op.length === 3) { okRef(op[1]); assertJsonValue(op[2]); }
else if (op.length === 2) assertJsonValue(op[1]);
else throw new TypeError("s arity"); return;
case "d": if (op.length === 2) okRef(op[1]); else if (op.length !== 1) throw new TypeError("d arity"); return;
case "a": if (op.length === 3) { okRef(op[1]); if (typeof op[2] !== "string") throw new TypeError("a value"); }
else if (op.length === 2) { if (typeof op[1] !== "string") throw new TypeError("a value"); }
else throw new TypeError("a arity"); return;
case "t": if (op.length === 3) { okRef(op[1]); if (!Number.isInteger(op[2])) throw new TypeError("t count"); }
else if (op.length === 2) { if (!Number.isInteger(op[1])) throw new TypeError("t count"); }
else throw new TypeError("t arity"); return;
case "p": {
const [i, r, items] = op.length === 5 ? [op[2], op[3], op[4]] : op.length === 4 ? [op[1], op[2], op[3]] : [];
if (items === undefined) throw new TypeError("p arity");
if (op.length === 5) okRef(op[1]);
if (!Number.isInteger(i) || (i as number) < 0) throw new TypeError("p index");
if (!Number.isInteger(r) || (r as number) < 0) throw new TypeError("p remove");
if (!Array.isArray(items)) throw new TypeError("p items");
for (const v of items) assertJsonValue(v);
return;
}
case "#": {
if (op.length !== 3 || !Number.isInteger(op[1]) || !Array.isArray(op[2])) throw new TypeError("# shape");
assertSafePath(op[2] as Path);
return;
}
// Silently skipping an unknown verb is how a newer producer's op vanishes.
default: throw new TypeError(`unknown op verb: ${String(verb)}`);
}
}
export function assertSafePath(path: Path): void {
for (const seg of path) {
if (typeof seg === "string") {
if (FORBIDDEN.has(seg)) throw new UnsafePathError(seg);
} else if (!Number.isInteger(seg) || seg < 0) {
throw new UnsafePathError(seg);
}
}
}
/**
* An index may address an existing element or append exactly one past the end.
*
* This is not an arbitrary cap — it is what keeps the value a `JsonValue`. A
* sparse array does not survive a JSON round trip: holes serialise to `null` and
* return as real properties, so `arr[7] = x` on a length-3 array already produces
* state a replica cannot match. Rejecting the write is more honest than silently
* diverging.
*
* It also removes the denial of service it would otherwise permit:
* `["s", ["xs", 4294967290], 1]` allocates a 4.29-billion-entry array from one op.
* Growth stays possible and stays proportional — the tracker already emits
* `arr.length = n` as a splice of explicit nulls, whose op size grows with the
* gap, so a large growth costs a large op rather than a small one.
*/
function assertIndexInRange(parent: readonly unknown[], index: number): void {
if (index > parent.length) throw new UnsafePathError(index);
}
// ─── Applier ─────────────────────────────────────────────────────────────────
export class PathError extends Error {
readonly path: Path | number;
constructor(path: Path | number) {
super(`unresolvable path: ${JSON.stringify(path)}`);
this.path = path;
this.name = "PathError";
}
}
/**
* Apply ops to a plain mutable value. Returns the value, because `r` replaces it
* outright and cannot be done in place.
*
* Takes decoded ops. Path ids and omitted paths are a wire concern — run
* `decode` first if the ops came from a boundary.
*/
export function apply<T>(target: T, ops: readonly Op[]): T {
let root = target as unknown as JsonValue;
for (const op of ops) {
assertValidOp(op);
if (op[0] === "r") {
// Adopted, not copied. The consumer owns the batch it was handed.
//
// Fanning one batch out to several consumers in-process therefore makes
// their replicas alias each other. That is an ownership rule, not a
// defect: copy the batch at the fan-out point, or let each consumer
// decode its own. A batch that crosses a real boundary is already
// distinct, because serialisation produces fresh objects.
root = op[1];
continue;
}
const path = op[1];
assertSafePath(path);
if (op[0] === "p") {
const target_ = path.length === 0 ? root : resolve(root, path);
if (!Array.isArray(target_)) throw new PathError(path);
target_.splice(op[2], op[3], ...op[4]);
continue;
}
// s/d/a/t can never target the root — the type forbids it.
const parent = resolve(root, path.slice(0, -1)) as Record<Seg, JsonValue>;
const key = path[path.length - 1]!;
if (Array.isArray(parent) && typeof key === "number") assertIndexInRange(parent, key);
// defineProperty rather than assignment: a setter inherited from the prototype
// chain would otherwise run on write.
const write = (value: JsonValue) => {
Object.defineProperty(parent, key, { value, writable: true, enumerable: true, configurable: true });
};
const read = (): unknown => (Object.hasOwn(parent, key) ? parent[key] : undefined);
switch (op[0]) {
case "s": write(op[2]); break;
case "d": Array.isArray(parent) ? (parent as unknown as JsonValue[]).splice(Number(key), 1) : delete parent[key]; break;
case "a": write(`${String(read() ?? "")}${op[2]}`); break;
case "t": write(String(read() ?? "").slice(op[2])); break;
}
}
return root as unknown as T;
}
function resolve(root: JsonValue, path: Path): JsonValue {
let node: JsonValue = root;
for (const seg of path) {
if (!isObj(node)) throw new PathError(path);
// Own properties only: an inherited getter must not run, and a walk must not
// escape the value into the prototype chain.
if (!Object.hasOwn(node, seg as PropertyKey)) throw new PathError(path);
node = (node as Record<Seg, JsonValue>)[seg]!;
}
if (!isObj(node)) throw new PathError(path);
return node;
}
// ─── Codec ───────────────────────────────────────────────────────────────────
//
// Path interning and arity omission, per `delta.md` §4. These live between the
// tracker and a boundary; `Op` and `apply` know nothing about them.
//
// ONE PAIR PER STREAM. The table spans an entire subscription or file: a path
// interned in batch 3 is referenced in batch 40. A second consumer that
// subscribes at batch 40 has never seen the definition, so it needs its own
// encoder — sharing one across consumers hands the late subscriber ids it cannot
// resolve. A base batch does not rescue it, because `["r", value]` carries no
// path refs and leaves the table empty.
const pathKey = (path: Path): string => path.join("\u0000");
export interface Encoder {
encode(ops: readonly Op[]): WireOp[];
}
/**
* Intern on SECOND use. A definition costs more than the path it replaces, so
* interning on first use loses on the many paths written exactly once —
* measured worse than no interning at all (`delta.md` §4.1).
*/
export function encoder(): Encoder {
const seen = new Set<string>();
const ids = new Map<string, number>();
let nextId = 0;
let previous: string | undefined; // last path in THIS batch
return {
encode(ops) {
// Arity omission is scoped to a batch. Letting it span batches would make
// a batch's first op depend on the previous batch's last one, so a reader
// that skips or reorders a batch decodes into the wrong path. Ids are the
// only cross-batch state, and the dictionary makes those explicit.
previous = undefined;
const out: WireOp[] = [];
for (const op of ops) {
if (op[0] === "r") {
out.push(op);
// A base batch is a RECOVERY POINT: a reader replays from the last one
// with a fresh decoder. So everything after it must be self-contained.
// Keeping ids across a replacement emits references to definitions the
// reader never saw — recovery fails with an unresolvable path id.
seen.clear();
ids.clear();
nextId = 0;
previous = undefined;
continue;
}
const path = op[1];
const key = pathKey(path);
// Same path as the previous op: drop the ref entirely.
if (key === previous) {
switch (op[0]) {
case "s": out.push(["s", op[2]]); break;
case "d": out.push(["d"]); break;
case "a": out.push(["a", op[2]]); break;
case "t": out.push(["t", op[2]]); break;
case "p": out.push(["p", op[2], op[3], op[4]]); break;
}
continue;
}
let ref: PathRef = path;
const existing = ids.get(key);
if (existing !== undefined) {
ref = existing;
} else if (seen.has(key)) {
const id = nextId++;
ids.set(key, id);
out.push(["#", id, path]); // second use: define, then reference
ref = id;
} else {
seen.add(key); // first use: inline
}
switch (op[0]) {
case "s": out.push(["s", ref as PathRef<NonEmptyPath>, op[2]]); break;
case "d": out.push(["d", ref as PathRef<NonEmptyPath>]); break;
case "a": out.push(["a", ref as PathRef<NonEmptyPath>, op[2]]); break;
case "t": out.push(["t", ref as PathRef<NonEmptyPath>, op[2]]); break;
case "p": out.push(["p", ref, op[2], op[3], op[4]]); break;
}
previous = key;
}
return out;
},
};
}
export interface Decoder {
decode(wire: readonly WireOp[]): Op[];
}
export function decoder(): Decoder {
const paths = new Map<number, Path>();
return {
decode(wire) {
let previous: Path | undefined; // scoped to the batch, as in encode
const out: Op[] = [];
for (const op of wire) {
assertValidWireOp(op);
if (op[0] === "#") {
assertSafePath(op[2]);
paths.set(op[1], op[2]);
continue;
}
if (op[0] === "r") {
out.push(op);
previous = undefined;
continue;
}
// Arity tells us whether a ref is present: the short forms omit it.
const short =
(op[0] === "d" && op.length === 1) ||
(op[0] !== "d" && op[0] !== "p" && op.length === 2) ||
(op[0] === "p" && op.length === 4);
let path: Path;
if (short) {
if (previous === undefined) throw new PathError([]);
path = previous;
} else {
const ref = op[1] as PathRef;
if (typeof ref === "number") {
const resolved = paths.get(ref);
if (resolved === undefined) throw new PathError(ref);
path = resolved;
} else {
path = ref;
}
previous = path;
}
switch (op[0]) {
case "s": out.push(["s", path as NonEmptyPath, (short ? op[1] : op[2]) as JsonValue]); break;
case "d": out.push(["d", path as NonEmptyPath]); break;
case "a": out.push(["a", path as NonEmptyPath, (short ? op[1] : op[2]) as string]); break;
case "t": out.push(["t", path as NonEmptyPath, (short ? op[1] : op[2]) as number]); break;
case "p": {
const [i, r, items] = short
? [op[1] as number, op[2] as number, op[3] as JsonValue[]]
: [op[2] as number, op[3] as number, op[4] as JsonValue[]];
out.push(["p", path, i, r, items]);
break;
}
}
}
return out;
},
};
}

113
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.bench.ts vendored
View File

@@ -1,113 +0,0 @@
/**
* Not a test — a guard against the cliffs found during implementation. Reports
* units you can reason about: cost per WRITE, and bytes on the wire relative to
* bytes consumed.
*
* npx vitest bench delta.bench.ts
*
* The headline finding: cost is per write, not per byte. A 100-byte chunk costs
* about the same as a 1 KB chunk, because the fixed cost is the window slice and
* the overlap probe. So chunk size — set by the exec env's coalescing interval,
* not by this package — decides both throughput and wire size.
*/
import { bench, describe } from "vitest";
import { decoder, encoder, track } from "./delta-impl.ts";
const WORDS = "the quick brown fox jumps over lazy dog while considering strategies".split(" ");
const text = (n: number): string => {
let s = "";
while (s.length < n) s += `${WORDS[Math.floor(Math.random() * WORDS.length)]} `;
return s.slice(0, n);
};
const CAP = 50_000;
const WRITES = 500;
const FLUSH_EVERY = 20;
/** One write = one chunk handed to the sink. Returns nothing; bench times it. */
function drive(initial: string, chunkSize: number, cap = CAP): void {
const t = track({ out: initial });
for (let i = 0; i < WRITES; i++) {
const next = t.state.out + text(chunkSize);
t.state.out = next.length > cap ? next.slice(next.length - cap) : next;
if (i % FLUSH_EVERY === FLUSH_EVERY - 1) t.flush();
}
t.flush();
}
/**
* Measured on the reference machine, 500 writes each. Reproduced here so a
* regression is legible rather than just "slower".
*
* append only, 100 B 127 µs/write 8k writes/s 101% of input on wire
* rolling, 100 B 176 µs/write 6k writes/s 135% of input on wire
* rolling, 1 KB 68 µs/write 15k writes/s 104% of input on wire
* rolling, 8 KB 366 µs/write 3k writes/s 31% of input on wire
* pathological, 100 B 307 µs/write 3k writes/s 727% of input on wire
*
* Two things to notice.
*
* 135% at 100-byte chunks: past the cap every write emits `truncate` + `append`,
* and two op envelopes exceed a 100-byte payload — we put MORE on the wire than we
* consumed. At 8 KB it is 31%. Flush granularity is therefore a size lever, not
* just an update-rate lever, and the exec env's coalescing is what keeps chunks
* off the bad end of that curve.
*
* The pathological row is bounded only because the overlap candidate scan is
* capped. Unbounded it was 4.7 s for 2000 writes against 93 ms; the cap makes it
* give up and emit a set — larger, never wrong.
*/
describe("rolling window, 50 KB cap", () => {
bench("100 B chunks", () => { drive(text(CAP), 100); });
bench("1 KB chunks", () => { drive(text(CAP), 1024); });
bench("8 KB chunks", () => { drive(text(CAP), 8192); });
bench("pathological: one repeated character", () => { drive("x".repeat(CAP), 100); });
});
describe("below the cap", () => {
// The dominant case for most tools: startsWith fast path, no probe at all,
// and one append per flush rather than a truncate/append pair.
bench("append only, never fills", () => { drive("", 100, Number.MAX_SAFE_INTEGER); });
});
// ── Codec: what path interning and arity omission actually save ────────────
//
// Reported as wire bytes relative to inline paths, on three shapes:
// - one hot path (a rolling tool-output window): omission does the work
// - a few alternating paths (lane state): interning does the work
// - many distinct paths: neither helps, and definitions cost a little
describe("codec", () => {
const streams = {
"one hot path": () => {
const t = track({ content: [{ type: "text", text: "" }] });
return Array.from({ length: 200 }, (_, i) => {
(t.state.content[0] as { text: string }).text += `line ${i} of build output\n`;
return t.flush();
});
},
"four alternating paths": () => {
const t = track({ a: "", b: "", c: "", d: "" });
return Array.from({ length: 200 }, (_, i) => {
for (const k of ["a", "b", "c", "d"] as const) t.state[k] += `${i}`;
return t.flush();
});
},
"200 distinct paths": () => {
const root: Record<string, number> = {};
for (let i = 0; i < 200; i++) root[`f${i}`] = 0;
const t = track(root);
return Array.from({ length: 5 }, (_, r) => {
for (let i = 0; i < 200; i++) t.state[`f${i}`] = r + 1;
return t.flush();
});
},
};
for (const [label, build] of Object.entries(streams)) {
bench(label, () => {
const enc = encoder();
const dec = decoder();
for (const ops of build()) dec.decode(enc.encode(ops));
});
}
});

133
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.examples.ts vendored
View File

@@ -1,133 +0,0 @@
/**
* Runnable illustrations of the two things that are easy to get wrong.
*
* node --experimental-strip-types delta.examples.ts
*
* Nothing here is a test — `delta.test.ts` covers the same ground with
* assertions. This exists to be read.
*/
import { apply, decoder, encoder, isBase, track, type Op } from "./delta-impl.ts";
const show = (label: string, ops: Op[]) => {
console.log(` ${label.padEnd(22)} ${ops.length} op(s), base=${isBase(ops)}`);
for (const op of ops) console.log(` ${trunc(JSON.stringify(op), 66)}`);
};
const trunc = (s: string, n: number) => (s.length <= n ? s : `${s.slice(0, n)}…`);
// ─────────────────────────────────────────────────────────────────────────
// 1. Replacing the whole value
// ─────────────────────────────────────────────────────────────────────────
console.log("\n1. REPLACING THE WHOLE VALUE\n");
const before = { user: { id: "u1", name: "ada" }, items: ["a", "b"], note: "n".repeat(120) };
const after = { user: { id: "u2", name: "bob" }, items: ["x"], note: "m".repeat(120) };
{
// The wrong way. `state` is a property of the TRACKER, not of the tracked
// object. Without a setter this swaps the proxy for a plain object and every
// later mutation is silently untracked — no error, no ops.
//
// The implementation defines `state` as a setter precisely so this works.
const t = track<Record<string, unknown>>(structuredClone(before));
t.flush(); // drain the opening base batch
t.state.items = ["changed first"]; // recorded…
t.state = structuredClone(after); // …then discarded: it described a dead value
show("assign to .state", t.flush());
t.state.user = { id: "u3" }; // still tracked afterwards
show(" and still tracked", t.flush());
}
{
// Dead-op elimination: a field written three times emits one op, and a parent
// replaced after its child emits one. Object key order is not preserved by
// this pass — see delta.md §5.1.
const t = track<Record<string, any>>(structuredClone(before));
t.flush(); // drain the opening base batch
t.state.items = ["first"];
t.state.items = ["second"];
t.state.user.name = "dropped";
t.state.user = { id: "u9" };
show("4 writes, 2 survive", t.flush());
}
// ─────────────────────────────────────────────────────────────────────────
// 2. Why checkpointing exists
// ─────────────────────────────────────────────────────────────────────────
console.log("\n2. RECOVERY LENGTH\n");
/**
* A bash-shaped workload: output arrives continuously and the sink writes
* durably on an interval. Each flush is one batch appended to a list.
* On resume the reader folds back to the last BASE batch, so that distance is
* the cost of recovery.
*/
function simulate(writes: number, checkpointEvery: number | null) {
const CAP = 50_000;
const t = track({ out: "" });
const isBaseBatch: boolean[] = [];
for (let i = 0; i < writes; i++) {
const next = `${t.state.out}[${i}] cc -c src/file${i}.c -o build/file${i}.o\n`;
t.state.out = next.length > CAP ? next.slice(next.length - CAP) : next;
if (checkpointEvery !== null && i % checkpointEvery === checkpointEvery - 1) t.rebase();
const ops = t.flush();
if (ops.length > 0) isBaseBatch.push(isBase(ops));
}
const lastBase = isBaseBatch.lastIndexOf(true);
return { written: isBaseBatch.length, replay: isBaseBatch.length - lastBase - 1 };
}
for (const [label, every] of [["never", null], ["rebase() every 50", 50]] as const) {
const r = simulate(500, every);
console.log(` ${label.padEnd(18)} ${r.written} batches written → ${r.replay} to replay on recovery`);
}
console.log(`
Nothing produces a base batch on its own: flush() emits ops, and a replacement
happens only when the producer asks for one. So a stream of appends stays a
stream of deltas indefinitely, and recovery grows without bound.
Two callers need rebase():
- pendingToolOutput, during a long-running command. Without it a ten-minute
build leaves hundreds of batches to fold on resume.
- a facet host on resubscribe. facets.md §9.2 requires the first batch of a
subscription to be a base batch, and the host must be able to produce one
on demand.
`);
// ─────────────────────────────────────────────────────────────────────────
// 3. A replica always agrees
// ─────────────────────────────────────────────────────────────────────────
console.log("3. PRODUCER AND REPLICA AGREE\n");
{
// The whole loop: tracker -> encode -> [boundary] -> decode -> apply.
const t = track<Record<string, unknown>>(structuredClone(before));
const enc = encoder();
const dec = decoder();
let replica: unknown;
const step = (label: string, mutate: () => void) => {
mutate();
const ops = dec.decode(enc.encode(t.flush()));
if (ops.length === 0) return;
// No base-batch branch: apply handles `r` by replacing, and tolerates an
// undefined target because a root op never reads it.
replica = apply(replica, ops);
const same = JSON.stringify(replica) === JSON.stringify(t.target);
console.log(` ${label.padEnd(22)} ${ops.length} op(s) ${isBase(ops) ? "BASE " : "delta"} matches: ${same}`);
};
step("opening base batch", () => {});
step("append to a string", () => { (t.state.note as string); t.state.note = `${t.state.note}!`; });
step("push", () => { (t.state.items as string[]).push("c"); });
step("nested set", () => { (t.state.user as Record<string, unknown>).name = "cy"; });
step("delete", () => { delete t.state.note; });
step("whole replacement", () => { t.state = structuredClone(after); });
step("rebase", () => { t.rebase(); });
}
console.log();

617
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.md vendored
View File

@@ -1,617 +0,0 @@
# Delta Tracking and the Op Vocabulary
> **Production status:** landed in `packages/chord/src/delta/index.ts`, with tests in `packages/chord/test/delta.test.ts`. Chord owns the dependency-free `Op`/`WireOp`, tracker, applier, codec, and validation boundary; Session storage, the Harness, and facets consume it. The implementation and tests beside this document are historical prototype and benchmark evidence, not production source.
>
> The landed tracker computes deltas at `flush()` from a dirty tree and baseline rather than retaining one op per mutation. That resolves FINDINGS D1. Production remeasurement also closes FINDINGS D2: the generic string path is below surrounding replication/rendering costs, so an explicit append/truncate API is rejected ([decision](append-decision.md)).
One mechanism covers assistant partials, tool output, tool details, lane state, and arbitrary facet state — on the wire and in durable storage.
```bash
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/delta.test.ts
# run from packages/chord
```
## 1. Why not an existing library
Immer, Valtio, Mutative and Colyseus all **record effects, not intent**. A string
is a leaf, so `text += chunk` is one write of the whole new string; there is no
representation in which the library could know you appended. `unshift` is the
same story — the engine shifts every index, so each is a write.
Measured, all four:
| operation | what they emit |
| --- | --- |
| `text += "x"` on 40 KB | `replace` carrying 40 KB |
| `arr.unshift(x)` on 100 items | ~100 index writes |
| `arr.pop()` | a write to a `length` path, which is not a document location |
Immer's own pitfalls page states the guarantee: patches are correct and
**explicitly not minimal**. Colyseus documents the same array weakness — removing
the first of 20 items costs 38 extra bytes.
Diffing base against result ourselves does not fix it either. A differ sees two
values, not intent. Prefix comparison catches pure growth, but a rolling window
that drops from the front *and* appends is neither a prefix nor a suffix of the
old value; measured on 5000 elements the fallback was
`{index: 0, remove: 5000, items: 5000}` — a full replace, in exactly the case the
optimisation existed for.
So the win is not a blind whole-value differ. The tracker records which paths and array operations became dirty, then compares only those subtrees against its accepted baseline at `flush()`.
## 2. Ops
**Tuples are the form. Everywhere. There is no keyed variant.**
```ts
export type Seg = string | number;
export type Path = readonly Seg[];
/** Non-empty: `s`/`d`/`a`/`t` cannot target the root. Enforced by the type. */
export type NonEmptyPath = readonly [Seg, ...Seg[]];
export type Op =
| readonly ["r", JsonValue] // replace the value
| readonly ["s", NonEmptyPath, JsonValue] // set
| readonly ["d", NonEmptyPath] // delete
| readonly ["a", NonEmptyPath, string] // append
| readonly ["t", NonEmptyPath, number] // truncate, in chars
| readonly ["p", Path, number, number, JsonValue[]] // splice: index, remove, items
```
Interning, id references and omitted paths are **not** here — they live in
`WireOp` and exist only between `encode` and `decode` (§4).
**`r` is the only op that replaces a whole value.** It is an op rather than a
frame kind for the same reason `a` and `t` exist: they are specialisations of `s`
that are smaller and state intent.
Do not encode a replacement as a set at the root path. Base-batch detection is a
**correctness boundary** — recovery stops there (§9) — so it must be a token
comparison, `ops[0]?.[0] === "r"`, and not a path inspection. A path-inspecting
predicate has to handle the two-element arity form, where `op[1]` is a value
rather than a path, and misclassifies `["s", []]`.
**Only `p` may target the root**, and only because a tracked value can itself be
an array — `entries.push(x)` on a root array is `["p", [], 3, 0, [x]]`. A `p`
covering its entire target is normalised at flush time to `r` (root) or `s`
(nested), so a root `p` is always a partial modification. The other four verbs
take a `NonEmptyPath`: `["s", [], v]` and `["d", []]` do not typecheck.
**Do not add a keyed in-memory shape with a codec at the serialization boundary.**
Two representations of one thing means two appliers, two size estimates, and a
conversion nobody needs. Ops are tuples in memory, on the wire, and on disk;
readability is what a debug formatter is for.
Six verbs. No `move`, `copy` or `test`: the first two are size optimisations for
a case we do not have, and `test` belongs to a conflict model we do not have,
since there is exactly one authoritative writer.
`chars` counts UTF-16 code units, matching `String.prototype.slice`. **Not bytes.**
Byte caps are a producer concern and never cross a boundary.
The format is JSON-Patch-*shaped*, not RFC 6902 conformant. That costs nothing:
RFC 6902 has had no successor since 2013, still has the same six ops, and nothing
anywhere adds a string splice. Immer was never conformant either.
## 3. The tracker
State is **plain TypeScript**. No containers, no handles, no schema, no
decorators. You mutate normally.
```ts
const t = track(laneView);
t.state.operation.streamingMessage.content[0].text += delta; // -> append
t.state.transcript.push(entry); // -> splice
t.state.tools[0].details.failures.push({ name, msg }); // -> splice
delete t.state.config.model; // -> delete
const ops = t.flush();
```
Three mechanisms, one per case:
**Objects** — ordinary `set` / `deleteProperty` traps.
**Arrays — mutator methods intercepted in the `get` trap.** `arr.push(x)` passes
through `get(arr, "push")` first, so the tracker returns its own function that
records `splice(len, 0, [x])` and then delegates. Intent is captured *before* the
engine performs its index writes, which is why `unshift` is one op rather than
O(n). This is the thing no other library does.
**Strings — dirty marking on the `set` trap, overlap detection at `flush()`.** Strings are primitives, so there is nothing to intercept. Given the accepted `prev` and final `next`, find the longest suffix of `prev` that is a prefix of `next`:
```
overlap === prev.length -> append(next.slice(overlap))
overlap > 0 -> truncate(prev.length - overlap) + append(rest)
overlap === 0 -> set(next)
```
Always correct, because the overlap is **verified**, not guessed. The rolling
window works:
```ts
s.out = s.out.slice(4) + "!!!";
// ["truncate", ["out"], 4]
// ["append", ["out"], "!!!"]
```
### 3.1 Overlap must not use a hand-written KMP
A KMP failure function is asymptotically right and **47x slower in practice**,
because it runs character by character in JS. Measured on 2000 window slides over
a 50 KB string: 2870 ms for KMP, 61 ms for a native probe.
The probe: `indexOf` a short head of `next` within `prev`, verify each candidate
with `endsWith`. Both are native. Verified identical to KMP across 200 window
slides.
A `startsWith` fast path runs first, so a pure append — the dominant case — skips
the probe entirely.
### 3.2 Rules the implementation must hold
**Inserted values are adopted; emitted payloads are cloned at flush.** Callers may retain read-only references but must not mutate adopted objects outside the tracker. `flush()` clones payloads and advances a cloned accepted baseline, so emitted ops never alias producer state.
**`x = undefined` normalises to `delete`.** JSON has no `undefined`, and a `set`
with a missing value is indistinguishable from a lost value after a
`JSON.stringify` round trip.
**`arr.length = n` is translated, never emitted as a path.** Shrink becomes a
splice, grow becomes a splice of nulls. `length` is a real mutation but not a
document location — this is the bug Immer ships (issue 208). Dropping it silently,
as an early prototype did, means `arr.length = 0` never reaches the replica.
**Numeric array keys are normalised to numbers.** Proxy traps deliver `"0"`;
paths carry `0`. Valtio ships the string form and it is worse for both size and
comparison.
**Proxies are cached per object and path**, so `s.a === s.a` and proxies are not
rebuilt on every access.
**Scope is `JsonValue` only, and it must be CHECKED, not assumed.**
`structuredClone` is not a JSON check — it clones `Map`, `Set`, `Date` and
`RegExp` happily, and the result then differs from what `JSON.stringify` produces,
so producer and replica diverge silently. Assert the value at record time and
reject symbol keys. See §7.4 for what leaks without it.
**Exactly two ops reach the root**, and the applier's root branch must handle
both: `r` always, and `p` when the tracked value is itself an array.
`s`/`d`/`a`/`t` cannot, by type, which makes the branch's fallthrough provably
unreachable. Handling only `r` there passes every hand-written test and fails on
the first randomised sequence that produces a root splice.
**Two writes to one address inside a transaction chain.** When several writes to
the same address are emitted in one commit, each must be recorded against the
previous one's result, not against pre-transaction state. The applier replays them
in order; anything else applies the second against a stale base. This did not fire
in the workload that surfaced it — it would have corrupted silently when it did.
### 3.2.1 Replacing the whole value
Assigning to `state` replaces it:
```ts
tracker.state = next; // emits ["r", next]; discards ops recorded before it
```
`state` **must be a setter on the tracker**, not a plain property. Without one,
`tracker.state = next` swaps the proxy for a plain object and every later
mutation is silently untracked — no error, no ops, `target` unchanged. It is also
the first thing anyone tries.
Prior ops are discarded because they describe a value that no longer exists.
Only a *full* replacement collapses to `r`. Rewrite part of the value and the ops
survive, because they are genuinely cheaper:
```
partial rewrite 2 op(s), base=false
["s",["user"],{"id":"u2","name":"bob"}]
["s",["items"],["x"]]
```
### 3.2.2 The first flush is a base batch
`track()` opens a stream, and a consumer starts with nothing, so the first flush
always emits `["r", value]` — carrying any mutations made before it:
```ts
const t = track({ x: 0, l: [] });
t.state.x = 100;
t.state.l.push("xyz");
t.flush(); // [["r", { x: 100, l: ["xyz"] }]]
t.state.x = 101;
t.flush(); // [["s", ["x"], 101]]
```
Requiring the producer to remember a `rebase()` first would fail at runtime, in
the consumer, far from the mistake.
### 3.2.3 Forcing a base batch later
```ts
tracker.rebase(); // next flush is ["r", value]; value unchanged
```
Discarding the pending ops is correct: the proxy mutates the target directly, so
the value already carries them. `tracker.state = tracker.state` has the same effect in the landed implementation, but `rebase()` states the intent directly.
**Nothing produces a base batch on its own.** `flush()` emits ops; a replacement
happens only when the producer asks for one. So a stream of appends stays a
stream of deltas indefinitely.
Recovery replays from the last base batch (§9). From `delta.examples.ts`, a
bash-shaped workload of 500 durable writes into a 50 KB rolling window:
| | batches written | to replay on recovery |
| --- | --- | --- |
| never | 500 | **499** |
| `rebase()` every 50 | 500 | 0 |
Two callers need this:
- **A durable sink.** `pendingToolOutput` accumulates one batch per checkpoint for
as long as a command runs. A `make -j8` running for ten minutes otherwise leaves
hundreds of batches to fold on resume. Checkpointing every N makes "at most N
batches to replay" a policy rather than an accident.
- **A facet host on resubscribe.** `facets.md` §9.2 requires the first batch of
every subscription to be a base batch, and a consumer that sees a gap
resubscribes to get one. The host has to be able to produce one on demand rather
than wait for one to happen by chance.
### 3.3 Known gaps
- `sort` / `reverse` / `fill` / `copyWithin` mark the array dirty and emit the resulting structural/index changes rather than preserving the producer's method intent. Add a dedicated op only if a measured workload needs one.
- A manual index-shift loop (`for (…) a[i] = a[i+1]`) can still cost O(n) sets. Correct, not minimal, and unavoidable — the producer genuinely wrote every element.
- Rolling-window string assignment still runs overlap discovery at flush, deliberately: production measurements do not justify text-specific tracker state or API surface ([decision](append-decision.md)).
### 3.4 Constraints the string algorithm must hold
Each of these is cheap to get wrong in a way that passes hand-written tests.
**A fixed-length probe cannot find an overlap shorter than the probe.** The head
must actually occur in `a`: `"abcdefgh"` -> `"defghxyz"` overlaps by 5, and a
64-character head cannot occur in an 8-character string. Try a long head first,
then fall back to a one-character head. A test using only 50 KB strings, where
overlaps are huge, will pass against a broken implementation.
**Bound the candidate scan.** Repetitive output — a build log, or a run of one
character — makes a long head match at thousands of positions, each costing a full
`endsWith`. Unbounded, 2000 slides over a 50 KB window take 4.7 s against 93 ms
bounded. Exceeding the bound returns 0, which emits a set: larger, never wrong.
**Property tests before anything depends on this.** 3000 random mutation
round-trips find the root-splice hole in §3.2 on the first run; the hand-written
cases do not. `delta.test.ts` is the suite to port.
## 4. The codec
Two vocabularies, not one.
`Op` is what the tracker produces and `apply` consumes. **Paths are always
inline.** It knows nothing about a dictionary.
`WireOp` is what crosses a boundary. It adds exactly two compressions:
```
["#", id, path] defines an id, emitted on a path's SECOND use
a numeric PathRef references a previously defined id
a shortened tuple reuses the previous op's path; arity disambiguates
```
`encoder().encode(ops): WireOp[]` and `decoder().decode(wire): Op[]` are the only places either exists. Keeping them out of `Op` means `apply` has no id resolution, no `#`
case, and no previous-path state — three branches removed from the hot path —
and dirty-tree generation always works with inline paths before encoding.
`["r", value]` carries no path, so it encodes to itself. That is why `isBase`
works unchanged on either vocabulary.
### 4.1 Two rules that are easy to get wrong
**One encoder/decoder pair per stream.** The id table spans a whole subscription
or file: a path interned in batch 3 is referenced in batch 40. A second consumer
that subscribes at batch 40 has never seen the definition, so it needs its own
encoder. Sharing one across consumers hands the late subscriber ids it cannot
resolve, and a base batch does not rescue it — `["r", value]` carries no refs and
leaves the table empty.
**Reset the table on a base batch.** A reader replays from the *last* base batch
with a fresh decoder, so everything after one must be self-contained. Carrying
ids across a replacement emits references to definitions the reader never saw:
```
4 : [["a",0,"x4"],["a",1,"y4"]]
5 BASE: [["r",{…}]]
6 : [["a",0,"x6"]] <- id 0 was defined in batch 1
-> RECOVERY FAILED: PathError - unresolvable path: 0
```
**Arity omission is scoped to a batch.** Letting it span batches makes a batch's
first op depend on the previous batch's last one, so a reader that skips or
reorders a batch decodes into the wrong path. Ids are the only cross-batch state,
and the dictionary makes those explicit.
### 4.2 Intern on second use, not first
A definition costs more than the path it replaces, so interning on first use
loses on every path written exactly once — and most are. Measured on a bulk
workload: **255.5 KB first-use versus 179.6 KB with no interning at all.**
### 4.3 Measured
Wire bytes against inline paths, round-trip verified:
| stream | inline | wire | saved |
| --- | --- | --- | --- |
| one hot path (a rolling tool-output window) | 11,290 B | 7,538 B | **33.2%** |
| four alternating paths (lane state) | 14,160 B | 11,032 B | **22.1%** |
| 200 distinct paths | 16,455 B | 16,035 B | 2.6% |
Omission does the work in the first case, interning in the second. In the third
neither helps and the definitions cost a little — which is the case second-use
interning exists to bound.
## 5. Flush emits ops, and drops the dead ones
`flush()` computes ops for the dirty paths against the last accepted baseline. It does not retain one op per mutation, so repeated and interleaved writes stay bounded by changed state rather than write count. There is no size comparison or replacement heuristic: a replacement is something the producer asks for by assigning `state` or calling `rebase()`.
An earlier design compared op bytes against the value's serialised size and
replaced when ops were larger. It was removed. Measured against emitting ops
unconditionally, on six workloads, it changed the output in two — both requiring
hundreds of distinct paths to change in a single flush. Neither the harness nor
the replication path does that: `LaneSnapshot` is folded one event at a time, and
the widest case (`run_end`) touches four fields. The rule cost a per-flush
comparison, a running size estimate, and an invalidation rule for the ops that
could not maintain it.
### 5.1 Dirty-tree collapse
The landed tracker records only which subtrees are dirty. At flush it compares each dirty subtree's accepted baseline with its final value and emits the surviving structural change. Repeated writes to one field collapse naturally; alternating writes to two fields retain two dirty paths rather than one op per mutation; a parent replacement subsumes dirty descendants where the final comparison permits it. `packages/chord/test/delta.test.ts` pins the interleaved rolling-window case at at most three ops after 1,000 alternating writes.
This replaces the prototype's backwards dead-op pass and adjacent-only coalescer. Do not port those algorithms into Chord: flush-time generation is the D1 fix.
> **Object key order is not a replicated invariant.** Values round-trip, but delete-and-reinsert activity within one flush can produce a different insertion order on a replica. Do not hash or content-address a replicated value, and sort explicitly where display order matters.
## 6. There is no frame type
The logical batch is `Op[]`; transport and durable storage carry the statefully encoded `WireOp[]`. Do not wrap either batch.
**`seq` does not belong in the payload.** It would defend against a lossy
transport we do not have: a durable list element already carries `seq` from
storage, an in-process callback cannot skip, and SSE either delivers in order or
breaks — and a break means resubscribe, which means a base batch. A second copy of
storage's `seq` could only disagree with the first. The SSE binding stamps `id:`,
which is where transport metadata belongs.
**Nothing else needs wrapping either.** A `kind` discriminator is redundant once a
replacement is an op (§2), and the address already says which value a batch
belongs to. What would remain is a struct around an array.
A producer batch is just `Op[]`, and its encoded boundary form is `WireOp[]`. A **base batch** is one whose first op is `r`;
`isBase(ops)` is `ops[0]?.[0] === "r"`, exact rather than heuristic because flush
guarantees `r` appears at index 0 or not at all (§5).
Everything a frame used to do is now done by something that already existed:
| was on the frame | now |
| --- | --- |
| `seq` | the list element's `seq`, or the SSE `id:` |
| `kind: "replace"` | the `r` op |
| which value it belongs to | the address |
| "this is a snapshot" | the `"base"` storage tag |
**Resubscription is a base batch plus buffered batches** — the same thing the lane
adapter already does for the harness: snapshot first, then whatever accumulated
while the client caught up. There is no `Last-Event-ID` and no cross-subscription
resume; that would need a retained op log, which §5 deliberately does not keep.
A consumer that sees a gap, fails to resolve a path, or connects cold takes one
path: ask for a base batch. That is why there is no `Rebase` type.
The cost, stated plainly: **a consumer cannot detect staleness from the payload
alone.** It relies on the transport reporting a break. For SSE and in-process that
is sound. If a lossy or multiplexed transport ever appears, sequencing goes on that
transport's envelope — not back into the ops.
## 7. Safety
Ops arrive from a facet, a plugin compartment, or a tool whose details may echo
model output. **None of it is trusted input**, and the boundary where untrusted
data meets trusted machinery is where this design has to hold.
The reference point is CVE-2025-55182 — RCE in React Server Components, CVSS 10.0,
exploited in the wild. Their Flight protocol is a compact tagged wire format that
reconstructs structure, so the resemblance is real. The failure was *"fails to
validate the structure correctly... treats the fake object as genuine"*: a forged
Chunk resolved as a Promise and exposed internal state containing gadgets to reach
`Function`.
**We are structurally safer, and not because of diligence:**
| | Flight | ops |
| --- | --- | --- |
| can describe runtime objects | yes — Chunks resolve as Promises | no |
| can reference code or modules | yes — client components | no |
| values | arbitrary object graphs | `JsonValue` |
| worst case from a forged payload | RCE | corrupted replica state |
Flight *must* reference code; that is its job. An op can only put a `JsonValue` at
a path, so there is no first link in a gadget chain. This is why an attacker cannot
plant a non-resolving `then` on `Object.prototype`: a data-only `then` is not
callable, and `await` resolves normally when `then` is not callable.
> **Rule: no op may name something the host resolves.** Not a mutation name, not a
> component id, not a module reference. Mutation names were removed for an
> unrelated reason (§8); this is the second and better reason never to reintroduce
> them, because it is the ingredient that made Flight exploitable.
### 7.1 Paths are the dangerous part
`JSON.parse` is safe on its own — `{"__proto__":{}}` becomes an *own* property.
The hazard is `parent[key]`, which is exactly what applying a path does.
A single key is not enough. `x["__proto__"] = v` swaps *x's own* parent, which is
local. It takes a **walk** — `x["__proto__"]["polluted"] = v` — to reach the shared
prototype, and a path is precisely a walk.
`constructor` is worse than `__proto__`, because `({}).constructor.constructor` is
`Function`. That ladder is closed here only because op values cannot be functions;
it is closed properly by rejecting the segment.
**Reserved path segments: `__proto__`, `constructor`, `prototype`.** Rejected at
record time *and* at apply time, including through an interned path id.
They are reserved as *segments*, not as values. An object with a literal
`"__proto__"` key replicates fine as a whole value; only a path walking *through*
it is refused. This is a genuine restriction on what is mutable — document it, do
not pretend it away.
Two further measures on the same hazard:
- **Write with `Object.defineProperty`**, never assignment, so an inherited setter
cannot run.
- **Resolve own properties only** (`Object.hasOwn`), so a walk cannot escape into
the prototype chain and an inherited getter cannot fire. This blocks
`constructor` a second time, since it is inherited.
What prototype pollution actually buys an attacker here is narrower than it first
appears: **it flips defaults, it does not overwrite explicit values.** An own
property shadows. So `{name:"bob", isAdmin:false}` is unaffected; an option bag read
as `opts.skipSandbox` is not. Our own `ShellExecOptions`, `ShellOutputCaptureOptions`,
`ListReadOptions` and `TrackerOptions` are exactly that shape. Note also that
`"x" in {}` and `const {x = false} = {}` both lie under pollution — which is why
the applier uses `Object.hasOwn`.
### 7.2 Array indices
An index may address an existing element or append exactly one past the end.
This is not an arbitrary cap; it is what keeps the value a `JsonValue`. A sparse
array does not survive a JSON round trip — holes serialise to `null` and return as
real properties — so `arr[7] = x` on a length-3 array already produces state a
replica cannot match. Rejecting the write is more honest than diverging.
It removes a denial of service as a side effect rather than as its purpose:
`["s",["xs",4294967290],1]` would otherwise allocate a 4.29-billion-entry array
from one op. Growth stays available and stays proportional, because
`arr.length = n` is emitted as a splice of explicit nulls whose op size grows with
the gap.
### 7.3 Validate op structure on decode
This is the RSC lesson applied to us. A decoder must not trust tuple shape.
Measured against an unvalidated applier:
| malformed op | result |
| --- | --- |
| `["p",["xs"],0,0,"not-an-array"]` | string spread into the array: `["n","o","t",…]` |
| `["s","a",9]` — path is a string | accepted; `"a".slice(0,-1)` is `""`, so it wrote at the root |
| `["ZZZ",["a"],9]` | silently ignored, replica diverges with no error |
Validate: verb is known, arity matches the verb, a path is an array of strings and
non-negative integers, `p` carries integer index and count plus an array of items,
`#` defines an array path. An unknown verb is an **error**, not a no-op — silently
skipping it is how a newer producer's op disappears and a replica drifts.
**One validator per vocabulary.** `assertValidOp` guards `apply` and rejects ids,
short forms and `#`; `assertValidWireOp` guards `decode` and permits them.
Validating a decoded op against the wire grammar is laxer than its own type: a
two-element `["s", value]` would pass, and `apply` would then read the value as a
path.
### 7.4 The tracker's cage leaks on types, not shape
A facet cannot forge an op: it mutates plain objects and the tracker builds the
tuples, so shape is well-formed by construction. Values and keys are another
matter, and `structuredClone` is **not** a JSON check.
| what a facet writes | what happens |
| --- | --- |
| a function | throws (`DataCloneError`) |
| a BigInt, a cycle | throws |
| `new Map([[1,2]])` | op carries `{}`, producer keeps a real Map — **silent divergence** |
| `new Date(0)` | op carries an ISO string, producer keeps a Date |
| `state[Symbol("s")] = 1` | emits `["s",[null],1]` — **a malformed op from our own tracker** |
So the tracker must check what it is given, at record time, where the mistake is:
- **Reject symbol keys.** They are not path segments.
- **Assert `JsonValue` on every recorded value.** Reject `Map`, `Set`, `Date`,
`RegExp`, typed arrays and class instances explicitly. The scope restriction in
§3.2 was a comment; it has to be a check.
A getter on state is safe — the trap records the computed result — and a facet
value that *looks* like an op is nested inside `["s", path, value]` and can never
be read as a top-level op, because nothing flattens.
### 7.5 Applier
```ts
export function apply<T>(target: T | undefined, ops: readonly Op[]): T;
```
Six verbs, no domain knowledge, no library, no tool code, no registry lookup, and
**no path table** — ids and omitted paths are resolved by `decode` before `apply`
sees anything (§4).
It returns the value rather than mutating in place, because `r` replaces it
outright.
Runs against a **plain mutable object the consumer owns**. It must not be pointed
at a value produced by Immer, which deep-freezes and would throw.
A page of code in any language, which is the property that keeps non-JS consumers
viable. Colyseus ships decoders for C#, Lua and Haxe on the same basis.
## 8. What this removes from the codebase
Concrete deletions, not simplifications in principle:
- **Immer**, from the harness and from the facet layer. See §1 for why a
patch-producing library cannot serve this purpose.
- **Per-type reducers.** There is one applier, `apply(target, ops)`, with no
domain knowledge and no registry. Nobody writes a fold for `ToolOutputState` or
for a plugin's shape.
- **`detailMutations` and `initialDetails`**, and with them the argument that
`details: unknown` forces special handling. A structural tracker never needs
the type.
- **`Rebase` as a reducer return value.** A fold that cannot apply leaves state
alone and the host sends a base batch.
- **Mutation names on the wire**, and therefore mutation-name version skew. Names
never cross a boundary, so adding or renaming one is not a breaking change.
And three things not to build, each of which looks reasonable until §1:
- **A blind whole-value differ.** The landed tracker compares only dirty subtrees against its accepted baseline; it does not scan unrelated state.
- **A keyed op shape plus a codec.** Tuples are the form everywhere (§2).
- **A frame wrapper.** What travels is `Op[]` (§6).
## 9. Durable form
A tracked value is stored as a **list of encoded `WireOp[]` batches**, one appended per flush. Base batches carry the storage tag `"base"`; one stateful decoder per value decodes them before `apply`.
Recovery reads backwards to the last base batch and applies forward:
```ts
readList(address, { order: "desc", stopAtTag: "base", limit: 100 })
```
`stopAtTag` is a stop condition *within a page*: if no base batch is in the page,
the consumer pages again with the cursor. The tag lives on the storage record
beside `seq`, never inside the value, so storage never parses ops. See
[scopes.md](../02-scopes/scopes.md) §11.
This is what makes "a replacement truncates recovery" (§5) real rather than
aspirational — a reader stops at the last base batch instead of replaying from
the beginning.
## 10. Open questions
- Prefix interning (a trie over path heads) if a workload emerges with many
distinct paths sharing long prefixes. Second-use interning (§4) removes the
pathological case; this would go further.
- Whether array reordering needs a dedicated op; the landed tracker currently marks the array dirty and emits the resulting structural/index changes.
- Cross-language replicas: the applier is a page of code in any language, but the
interning tables and the wire framing are not specified for a non-JS consumer.
Address interning in the JSONL log is the same trick as path interning, applied
one layer up over `namespace` + `key`. It is **not** an open question — it is
specified in [scopes.md](../02-scopes/scopes.md) §12 and is a separate dictionary from this one.

563
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.test.ts vendored
View File

@@ -1,563 +0,0 @@
import { describe, expect, it } from "vitest";
import { apply, assertJsonValue, assertValidOp, assertValidWireOp, decoder, encoder, isBase, overlap, track, type Op, type WireOp } from "./delta-impl.ts";
const PAD = "p".repeat(400); // see "adaptive emission": a delta must be able to win
describe("tracker: intent", () => {
it("records an append, not a replacement", () => {
const t = track({ s: "", pad: PAD });
t.flush(); // drain the base batch
t.state.s += "ab";
t.state.s += "cd";
const ops = t.flush();
expect(ops).toEqual([["a", ["s"], "abcd"]]);
expect(apply({ s: "", pad: PAD }, ops)).toEqual({ s: "abcd", pad: PAD });
});
it("recovers truncate+append from a rolling window", () => {
// The case every effect-recording library degrades to a whole-value set.
const t = track({ s: "abcdefgh", pad: PAD });
t.flush(); // drain the base batch
t.state.s = t.state.s.slice(3) + "xyz";
const ops = t.flush();
expect(ops).toEqual([["t", ["s"], 3], ["a", ["s"], "xyz"]]);
expect(apply({ s: "abcdefgh", pad: PAD }, ops)).toEqual({ s: "defghxyz", pad: PAD });
});
it("records array intent as one splice", () => {
const t = track({ xs: [1, 2], pad: PAD });
t.flush(); // drain the base batch
t.state.xs.push(3);
expect(t.flush()).toEqual([["p", ["xs"], 2, 0, [3]]]);
});
it("normalises undefined to a delete", () => {
const t = track({ a: 1, pad: PAD });
t.flush(); // drain the base batch
(t.state as Record<string, unknown>).a = undefined;
expect(t.flush()).toEqual([["d", ["a"]]]);
});
});
describe("tracker: root ops", () => {
it("splices a value that is itself an array", () => {
const t = track([1, 2, 3, PAD]);
t.flush(); // drain the base batch
t.state.push(4);
const ops = t.flush();
expect(ops).toEqual([["p", [], 4, 0, [4]]]);
expect(apply([1, 2, 3, PAD], ops)).toEqual([1, 2, 3, PAD, 4]);
});
it("normalises a splice covering the whole root to a replacement", () => {
const t = track([1, 2, 3]);
t.flush(); // drain the base batch
t.state.splice(0, 3, 9);
const ops = t.flush();
expect(ops).toEqual([["r", [9]]]);
expect(isBase(ops)).toBe(true);
});
it("normalises a nested splice-all to a set, not a replacement", () => {
const t = track({ xs: [1, 2, 3], pad: PAD });
t.flush(); // drain the base batch
t.state.xs.splice(0, 3, 9);
expect(t.flush()).toEqual([["s", ["xs"], [9]]]);
});
it("normalises length = 0 on the root", () => {
const t = track([1, 2, 3]);
t.flush(); // drain the base batch
t.state.length = 0;
const ops = t.flush();
expect(ops).toEqual([["r", []]]);
expect(apply([1, 2, 3], ops)).toEqual([]);
});
});
describe("replacing the whole value", () => {
// The proxy cannot observe `tracker.state = next`: `state` belongs to the
// tracker, not to the tracked object, so assigning to it swaps the proxy for a
// plain object and silently stops tracking. Replacement must be a method.
it("assigning to state emits a base batch and keeps tracking", () => {
const t = track<Record<string, unknown>>({ p: 1, q: PAD });
t.flush(); // drain the base batch
t.state.p = 2;
t.state = { r: 9, s: "new" };
const ops = t.flush();
expect(ops).toEqual([["r", { r: 9, s: "new" }]]);
expect(isBase(ops)).toBe(true);
expect(apply({}, ops)).toEqual(t.target);
t.state.r = 10; // the new value is tracked
expect(t.flush()).toEqual([["s", ["r"], 10]]);
});
it("assigning to state discards ops recorded before it", () => {
// They describe a value that no longer exists.
const t = track<Record<string, unknown>>({ p: 1, q: PAD });
t.flush(); // drain the base batch
t.state.p = 2;
t.state = { z: 1 };
expect(t.flush()).toEqual([["r", { z: 1 }]]);
});
it("a partial rewrite keeps its ops", () => {
const t = track<Record<string, unknown>>({ a: "x".repeat(200), b: "y".repeat(200) });
t.flush(); // drain the base batch
t.state.a = "p";
const ops = t.flush();
expect(isBase(ops)).toBe(false);
expect(ops).toEqual([["s", ["a"], "p"]]);
});
it("rebase() forces a base batch without changing the value", () => {
// This is the checkpoint. Recovery replays from the last base batch, so a
// producer must be able to bound that rather than depend on the adaptive
// size rule, which compares each flush independently.
const t = track<Record<string, unknown>>({ p: 1, pad: PAD });
t.flush(); // drain the base batch
t.rebase();
const ops = t.flush();
expect(isBase(ops)).toBe(true);
expect(apply({}, ops)).toEqual(t.target);
});
it("rebase() applies once, not to every later flush", () => {
const t = track<Record<string, unknown>>({ p: 1, pad: PAD });
t.flush(); // drain the base batch
t.rebase();
t.flush();
t.state.p = 2;
expect(t.flush()).toEqual([["s", ["p"], 2]]);
});
});
describe("the first flush", () => {
it("is always a base batch", () => {
// A consumer starts with nothing, so a stream opening with deltas has
// nothing to apply them to. Requiring the producer to remember would fail
// at runtime, in the consumer, rather than where the mistake is.
const t = track<Record<string, unknown>>({ x: 0 });
t.state.x = 100;
const ops = t.flush();
expect(isBase(ops)).toBe(true);
expect(ops).toEqual([["r", { x: 100 }]]); // carries mutations made before it
});
it("is followed by deltas", () => {
const t = track<Record<string, unknown>>({ x: 0, pad: PAD });
t.flush();
t.state.x = 1;
expect(t.flush()).toEqual([["s", ["x"], 1]]);
});
it("emits nothing when untouched", () => {
// An unmutated tracker has nothing to say; the base batch is owed to a
// consumer, and there is no consumer until someone flushes.
const t = track<Record<string, unknown>>({ x: 0 });
expect(t.flush()).toEqual([["r", { x: 0 }]]);
expect(t.flush()).toEqual([]);
});
});
describe("apply and fan-out", () => {
it("adopts an `r` payload rather than copying it", () => {
// The consumer owns the batch it was handed. Copying every base batch
// defensively doubles the memory of the one op carrying the whole value.
//
// The rule that follows: do not hand one batch to two in-process
// consumers. This test pins the aliasing so nobody "fixes" it by adding a
// clone back.
const t = track<Record<string, unknown>>({ n: 0 });
const batch = t.flush();
const a = apply(undefined, batch) as Record<string, unknown>;
const b = apply(undefined, batch) as Record<string, unknown>;
a.n = 1;
expect(b.n).toBe(1);
});
it("does not alias the producer", () => {
const t = track<Record<string, unknown>>({ x: 0 });
const replica = apply(undefined, t.flush()) as Record<string, unknown>;
t.state.x = 999;
t.flush();
expect(replica.x).toBe(0);
});
it("folds a whole stream without a base-batch branch", () => {
// `apply` handles `r` by replacing, and tolerates an undefined target,
// so a consumer needs no isBase check and no clone of its own.
const t = track<Record<string, any>>({ x: 0, l: [] as string[] });
const enc = encoder();
const dec = decoder();
let replica: unknown;
const send = () => { replica = apply(replica, dec.decode(enc.encode(t.flush()))); };
t.state.x = 100;
t.state.l.push("xyz");
send();
t.state.x = 101;
send();
expect(replica).toEqual(t.target);
});
});
describe("dead-op elimination", () => {
// Ops apply in order, so an op is dead if a later one overwrites the whole
// subtree it lives in. Only s/d/r dominate; a/t/p modify what is there.
const run = (mutate: (s: any) => void, init: any = { a: { b: 1 }, x: 1, y: 2 }) => {
const t = track<any>(structuredClone(init));
t.flush(); // drain the base batch
mutate(t.state);
return t.flush();
};
it("collapses repeated writes to one field", () => {
expect(run((s) => { s.x = 1; s.x = 2; s.x = 3; })).toEqual([["s", ["x"], 3]]);
});
it("drops a write superseded by a later, non-adjacent one", () => {
expect(run((s) => { s.x = 10; s.y = 20; s.x = 30; }))
.toEqual([["s", ["y"], 20], ["s", ["x"], 30]]);
});
it("drops a child write when the parent is replaced after it", () => {
expect(run((s) => { s.a.b = 99; s.a = { c: 5 }; })).toEqual([["s", ["a"], { c: 5 }]]);
});
it("keeps a child write that follows the parent replacement", () => {
expect(run((s) => { s.a = { b: 1 }; s.a.b = 7; }))
.toEqual([["s", ["a"], { b: 1 }], ["s", ["a", "b"], 7]]);
});
it("collapses set-then-delete to the delete", () => {
expect(run((s) => { s.x = 5; delete s.x; })).toEqual([["d", ["x"]]]);
});
it("keeps delete-then-set, to preserve key position", () => {
// Dropping the delete would leave the key in its original slot while the
// producer moved it to the end. Same value, different serialisation.
expect(run((s) => { delete s.x; s.x = 5; })).toEqual([["d", ["x"]], ["s", ["x"], 5]]);
});
it("is linear in the number of ops", () => {
// The naive formulation compares every op against every dominator, which
// is quadratic and degrades on exactly the wide flush this pass cleans up.
const wide = (n: number) => {
const root: Record<string, number> = {};
for (let i = 0; i < n; i++) root[`f${i}`] = i;
const t = track(root);
t.flush(); // drain the base batch
const started = performance.now();
for (let i = 0; i < n; i++) t.state[`f${i}`] = i + 1;
t.flush();
return performance.now() - started;
};
wide(200); // warm
const small = Math.max(wide(250), 0.1);
const large = wide(2500);
expect(large / small).toBeLessThan(40); // linear would be ~10x
});
it("collapses a pathological redundant producer", () => {
const t = track<any>({ a: { b: 1 }, x: 1 });
t.flush(); // drain the base batch
for (let i = 0; i < 2000; i++) { t.state.x = i; t.state.a.b = i; }
t.state.a = { done: true };
expect(t.flush()).toHaveLength(2);
});
});
describe("flush", () => {
it("drops everything before a replacement", () => {
const t = track([1, 2]);
t.flush(); // drain the base batch
t.state.push(3);
t.state.splice(0, 3, 7);
expect(t.flush()).toEqual([["r", [7]]]);
});
it("keeps the prefix when the replacement is nested", () => {
// `s` on a subtree is not a root replacement, so earlier ops stay live.
const t = track({ a: 1, xs: [1, 2, 3], pad: PAD });
t.flush(); // drain the base batch
t.state.a = 2;
t.state.xs.splice(0, 3, 9);
expect(t.flush()).toEqual([["s", ["a"], 2], ["s", ["xs"], [9]]]);
});
it("rejects a constructor walk", () => {
// ({}).constructor.constructor is Function — the classic escape ladder.
expect(() => apply({}, JSON.parse('[["s",["constructor","prototype","gadget"],true]]'))).toThrow();
expect(({} as Record<string, unknown>).gadget).toBeUndefined();
});
it("rejects a forbidden path reached through an interned id", () => {
const dict = new Map<number, readonly (string | number)[]>([[0, ["__proto__", "w"]]]);
expect(() => apply({}, [["s", 0, true]], (id) => dict.get(id)!)).toThrow();
});
it("does not run an inherited setter", () => {
Object.defineProperty(Object.prototype, "trap", {
get() { throw new Error("inherited getter ran"); },
configurable: true,
});
try {
expect(() => apply({}, [["s", ["trap"], 1]])).not.toThrow();
} finally {
delete (Object.prototype as Record<string, unknown>).trap;
}
});
it("allows a reserved name as a VALUE key", () => {
// Reserved as segments, not as values.
const out = apply({}, [["s", ["a"], JSON.parse('{"__proto__":{"z":1}}')]]);
expect(({} as Record<string, unknown>).z).toBeUndefined();
expect(out).toBeDefined();
});
});
describe("safety: array indices", () => {
// An index may address an existing element or append exactly one past the end.
// Not an arbitrary cap: a sparse array does not survive a JSON round trip, so
// arr[7] = x on a length-3 array already produces unreplicable state.
it("writes an existing index", () => {
expect(apply({ xs: [1, 2, 3] }, [["s", ["xs", 1], 9]])).toEqual({ xs: [1, 9, 3] });
});
it("appends one past the end", () => {
expect(apply({ xs: [1, 2, 3] }, [["s", ["xs", 3], 9]])).toEqual({ xs: [1, 2, 3, 9] });
});
it("rejects a gap", () => {
expect(() => apply({ xs: [1, 2, 3] }, [["s", ["xs", 5], 9]])).toThrow();
});
it("rejects a huge index", () => {
// Would otherwise allocate 4.29 billion entries from one op.
expect(() => apply({ xs: [] }, [["s", ["xs", 4_294_967_290], 1]])).toThrow();
});
it("allows explicit growth with nulls", () => {
expect(apply({ xs: [1] }, [["p", ["xs"], 1, 0, [null, null, 9]]])).toEqual({ xs: [1, null, null, 9] });
});
});
describe("safety: op structure", () => {
// The CVE-2025-55182 lesson: a decoder must not trust tuple shape.
it("rejects an unknown verb rather than skipping it", () => {
// Silently skipping is how a newer producer's op vanishes and a replica drifts.
expect(() => apply({ a: 1 }, [["ZZZ", ["a"], 9] as never])).toThrow();
});
it("rejects non-array splice items", () => {
// Unvalidated, this spreads a string: ["n","o","t","-","a",…]
expect(() => apply({ xs: [1] }, [["p", ["xs"], 0, 0, "not-an-array"] as never])).toThrow();
});
it("rejects a string path", () => {
// Unvalidated, "a".slice(0,-1) is "", so it resolved to the ROOT and wrote.
expect(() => apply({ a: 1 }, [["s", "a", 9] as never])).toThrow();
});
it("rejects a non-tuple op", () => {
expect(() => apply({ a: 1 }, [{ op: "s" } as never])).toThrow();
expect(() => apply({ a: 1 }, [null as never])).toThrow();
});
it("clamps a splice remove past the end, as Array.prototype.splice does", () => {
// Deterministic and identical on both sides — not a hole.
expect(apply({ xs: [1, 2] }, [["p", ["xs"], 0, 1e9, []]])).toEqual({ xs: [] });
});
});
describe("safety: what a facet can put on the wire", () => {
// A facet cannot forge an op — it mutates plain objects and the tracker builds
// the tuples. Values and keys are the leak, and structuredClone is NOT a JSON
// check: it clones a Map happily, and the op then carries {} while the producer
// keeps a real Map.
const rejects = (name: string, mutate: (s: Record<string, unknown>) => void) =>
it(`rejects ${name}`, () => {
const t = track<Record<string, unknown>>({ pad: PAD });
t.flush(); // drain the base batch
expect(() => mutate(t.state)).toThrow();
});
rejects("a function", (s) => { s.f = () => {}; });
rejects("a Map", (s) => { s.m = new Map([[1, 2]]); });
rejects("a Date", (s) => { s.d = new Date(0); });
rejects("a BigInt", (s) => { s.b = 1n; });
rejects("a cycle", (s) => { const a: Record<string, unknown> = {}; a.self = a; s.c = a; });
rejects("a symbol key", (s) => { s[Symbol("k") as unknown as string] = 1; });
it("allows a getter on state, recording the computed result", () => {
const t = track<Record<string, unknown>>({ pad: PAD });
t.flush(); // drain the base batch
Object.defineProperty(t.target, "g", { get: () => 42, enumerable: true });
t.state.x = 1;
expect(t.flush()).toEqual([["s", ["x"], 1]]);
});
it("accepts a value that merely looks like an op", () => {
// Nested inside ["s", path, value]; nothing flattens, so it cannot be read
// as a top-level op.
const t = track<Record<string, unknown>>({ pad: PAD });
t.flush(); // drain the base batch
t.state.x = ["r", { evil: true }];
expect(t.flush()).toEqual([["s", ["x"], ["r", { evil: true }]]]);
});
});
describe("assertions", () => {
it("assertJsonValue rejects non-JSON", () => {
for (const bad of [new Map(), new Set(), new Date(), /re/, () => {}, Number.NaN, Infinity]) {
expect(() => assertJsonValue(bad)).toThrow();
}
for (const good of [null, true, 1, "s", [1, "a"], { a: { b: [null] } }]) {
expect(() => assertJsonValue(good)).not.toThrow();
}
});
it("assertValidOp accepts decoded ops and rejects wire forms", () => {
// Validating Op against the wire grammar would be laxer than the type: a
// two-element ["s", value] would pass, and apply would read the value as a
// path. Each vocabulary gets the validator that matches it.
for (const op of [
["r", { a: 1 }], ["s", ["a"], 1], ["d", ["a"]],
["a", ["a"], "x"], ["t", ["a"], 2], ["p", ["a"], 0, 0, []],
] as never[]) {
expect(() => assertValidOp(op)).not.toThrow();
}
for (const wireOnly of [["s", 1], ["d"], ["a", "x"], ["t", 2], ["p", 0, 0, []], ["#", 0, ["a"]], ["s", 0, 1]] as never[]) {
expect(() => assertValidOp(wireOnly)).toThrow();
expect(() => assertValidWireOp(wireOnly)).not.toThrow();
}
});
});
describe("codec: path interning and arity omission", () => {
// One pair per stream. The table spans a whole subscription or file, so a
// second consumer joining later needs its own encoder.
const roundTrip = (batches: Op[][]) => {
const enc = encoder();
const dec = decoder();
return batches.map((ops) => dec.decode(enc.encode(ops)));
};
it("round-trips a stream exactly", () => {
const t = track<Record<string, any>>({ a: { deep: "" }, b: { deep: "" } });
t.flush(); // drain the base batch
const batches: Op[][] = [];
for (let i = 0; i < 6; i++) {
t.state.a.deep += `x${i}`;
t.state.b.deep += `y${i}`;
batches.push(t.flush());
}
expect(roundTrip(batches)).toEqual(batches);
});
it("interns on second use, not first", () => {
const enc = encoder();
const p = ["a", "deep"] as const;
const first = enc.encode([["a", p, "1"]]);
const second = enc.encode([["a", p, "2"]]);
expect(first).toEqual([["a", p, "1"]]); // inline
expect(second).toEqual([["#", 0, p], ["a", 0, "2"]]); // define, then use
});
it("omits the path when it repeats", () => {
const enc = encoder();
const p = ["a"] as const;
expect(enc.encode([["s", p, 1], ["s", p, 2]])).toEqual([["s", p, 1], ["s", 2]]);
});
it("resets the table on a base batch, so recovery is self-contained", () => {
// A reader replays from the LAST base batch with a fresh decoder. Ids
// defined before it were never seen, so keeping them breaks recovery with
// an unresolvable path id.
const enc = encoder();
const p = ["a", "deep"] as const;
enc.encode([["a", p, "1"]]);
enc.encode([["a", p, "2"]]); // id 0 assigned
const base = enc.encode([["r", { a: { deep: "x" } }]]);
const after = enc.encode([["a", p, "3"]]);
expect(base).toEqual([["r", { a: { deep: "x" } }]]);
expect(after).toEqual([["a", p, "3"]]); // inline again
const dec = decoder();
expect(() => dec.decode(base)).not.toThrow();
expect(dec.decode(after)).toEqual([["a", p, "3"]]);
});
it("survives recovery from the last base batch", () => {
const enc = encoder();
const t = track<Record<string, any>>({ a: { deep: "" }, b: { deep: "" } });
t.flush(); // drain the base batch
const wire: WireOp[][] = [];
for (let i = 0; i < 8; i++) {
t.state.a.deep += `x${i}`;
t.state.b.deep += `y${i}`;
if (i === 5) t.rebase();
wire.push(enc.encode(t.flush()));
}
const lastBase = wire.map(isBase).lastIndexOf(true);
const dec = decoder();
let replica: unknown;
for (const w of wire.slice(lastBase)) {
const ops = dec.decode(w);
replica = replica === undefined
? structuredClone((ops[0] as ["r", unknown])[1])
: apply(replica, ops);
}
expect(replica).toEqual(t.target);
});
it("round-trips random streams", () => {
for (let round = 0; round < 300; round++) {
const t = track<Record<string, any>>({ a: { p: "", q: "" }, b: [] as unknown[], c: 0 });
t.flush(); // drain the base batch
const batches: Op[][] = [];
for (let i = 0; i < 8; i++) {
const r = Math.random();
if (r < 0.3) t.state.a.p += "x";
else if (r < 0.5) t.state.a.q += "y";
else if (r < 0.65) t.state.b.push(i);
else if (r < 0.8) t.state.c = i;
else if (r < 0.9) delete t.state.c;
else t.rebase();
const ops = t.flush();
if (ops.length > 0) batches.push(ops);
}
expect(roundTrip(batches)).toEqual(batches);
}
});
});
describe("property: random round-trip", () => {
// 3000 random sequences found the root-splice hole on the first run, before any
// hand-written case did.
const rnd = (d = 0): unknown => {
const r = Math.random();
if (d > 2 || r < 0.3) return Math.floor(Math.random() * 5);
if (r < 0.45) return ["x", "y", null, true][Math.floor(Math.random() * 4)];
if (r < 0.7) return Array.from({ length: Math.floor(Math.random() * 4) }, () => rnd(d + 1));
const o: Record<string, unknown> = {};
for (const k of ["a", "b", "c"]) if (Math.random() < 0.6) o[k] = rnd(d + 1);
return o;
};
it("a replica always matches the producer", () => {
let checked = 0;
for (let i = 0; i < 3000; i++) {
const base = rnd();
if (typeof base !== "object" || base === null) continue;
const t = track(structuredClone(base) as object);
t.flush(); // drain the base batch
const next = rnd();
if (Array.isArray(t.state) && Array.isArray(next)) {
(t.state as unknown[]).splice(0, (t.state as unknown[]).length, ...next);
} else if (!Array.isArray(t.state) && typeof next === "object" && next !== null && !Array.isArray(next)) {
const s = t.state as Record<string, unknown>;
for (const k of Object.keys(s)) if (!(k in next)) delete s[k];
for (const [k, v] of Object.entries(next)) s[k] = v;
} else continue;
expect(apply(structuredClone(base), t.flush())).toEqual(t.target);
checked++;
}
// Most random pairs are shape-incompatible and skipped; this only guards
// against the loop silently checking nothing.
expect(checked).toBeGreaterThan(200);
});
});

285
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/01-delta/delta.ts vendored
View File

@@ -1,285 +0,0 @@
// ─────────────────────────────────────────────────────────────────────────────
// harness/delta — intent-recording change tracking over plain JSON.
//
// Depends on nothing else in the harness. Session storage, the runtime and the
// facet host consume it; keep the arrows pointing that way.
//
// State is plain TS. You mutate it normally. Ops come out carrying what you
// *did*, not what changed — `text += chunk` yields an append, not a 50 KB
// replacement; `arr.unshift(x)` yields one splice, not O(n) index writes.
// ─────────────────────────────────────────────────────────────────────────────
export type JsonValue = null | boolean | number | string | JsonValue[] | { [k: string]: JsonValue };
export type Seg = string | number;
export type Path = readonly Seg[];
/** Non-empty: `s`/`d`/`a`/`t` cannot target the root. Enforced by the type. */
export type NonEmptyPath = readonly [Seg, ...Seg[]];
/** A path inline on first use, an interned id thereafter. */
export type PathRef<P extends Path = Path> = P | number;
/**
* Tuples are the form — in memory, on the wire, on disk. There is no keyed
* variant and no codec between them.
*
* `r` is the ONLY op that replaces a whole value. It is an op rather than a
* frame kind for the same reason `a` and `t` exist: they are specialisations of
* `s` that are smaller and state intent. It also makes base detection a token
* comparison rather than a path inspection, which matters because recovery stops
* at the last base batch — a correctness boundary.
*
* Only `p` may target the root, and only because a tracked value can itself be
* an array. A `p` covering its entire target is normalised at RECORD time to `r`
* or `s`, so a root `p` is always a partial modification.
*
* The two-element forms reuse the previous op's path; arity disambiguates.
*/
export type Op =
| readonly ["r", JsonValue]
| readonly ["s", NonEmptyPath, JsonValue]
| readonly ["d", NonEmptyPath]
| readonly ["a", NonEmptyPath, string]
| readonly ["t", NonEmptyPath, number]
| readonly ["p", Path, number, number, JsonValue[]];
/**
* What crosses a boundary. Adds path interning and arity omission, and nothing
* else. `["r", value]` carries no path, so it encodes to itself — which is why
* `isBase` works on either vocabulary.
*/
export type WireOp =
| readonly ["r", JsonValue]
| readonly ["s", PathRef<NonEmptyPath>, JsonValue] | readonly ["s", JsonValue]
| readonly ["d", PathRef<NonEmptyPath>] | readonly ["d"]
| readonly ["a", PathRef<NonEmptyPath>, string] | readonly ["a", string]
| readonly ["t", PathRef<NonEmptyPath>, number] | readonly ["t", number]
| readonly ["p", PathRef, number, number, JsonValue[]]
| readonly ["p", number, number, JsonValue[]]
| readonly ["#", number, Path];
/**
* There is no frame type. What travels is `Op[]`.
*
* A **base batch** is one whose first op is `r`; the first batch of a stream is
* always one, and storage tags it "base" so recovery can stop there. Sequencing
* lives on the transport — a list element's `seq`, or an SSE `id:` — never in the
* payload. See `delta.md` §6.
*/
// ─── Safety ──────────────────────────────────────────────────────────────────
/**
* Path segments that reach the prototype chain. Rejected at record time AND at
* apply time, including through an interned id.
*
* Reserved as SEGMENTS, not as values: an object with a literal `__proto__` key
* replicates fine as a whole value; only a path walking through it is refused.
* That is a real restriction on what is mutable — see `delta.md` §7.1.
*/
export declare const RESERVED_SEGMENTS: ReadonlySet<string>;
export declare class UnsafePathError extends Error {
readonly segment: Seg;
}
/** Rejects reserved segments, symbol keys, and negative or non-integer indices. */
export declare function assertSafePath(path: Path): void;
/**
* Rejects anything `JSON.stringify` would not round-trip: Map, Set, Date, RegExp,
* typed arrays, class instances, functions, BigInt, cycles.
*
* `structuredClone` is NOT a JSON check — it clones a Map happily, and the op then
* carries `{}` while the producer keeps a real Map. Silent divergence.
*/
export declare function assertJsonValue(value: unknown): asserts value is JsonValue;
/**
* Validate an op's structure before applying it. A decoder must not trust tuple
* shape: an unvalidated applier spreads a string into an array given
* `["p",["xs"],0,0,"not-an-array"]`, and writes at the root given a string path.
*
* An unknown verb is an error, not a no-op — skipping it is how a newer producer's
* op vanishes and a replica drifts. This is the CVE-2025-55182 lesson
* (`delta.md` §7.3).
*/
export declare function assertValidOp(op: unknown): asserts op is Op;
/**
* The same, for the wire grammar, where ids and short forms are legal.
* `decode` uses this; `apply` uses `assertValidOp`. Validating a decoded op
* against the wire grammar is laxer than its type — a two-element `["s", value]`
* would pass and then be read as a path.
*/
export declare function assertValidWireOp(op: unknown): asserts op is WireOp;
// ─── Classification ─────────────────────────────────────────────────────────
//
// Ops are tuples, so classification is easy to hand-roll and easy to get wrong.
// `op[0] === "s"` is not enough — a set on any path is "s"; what makes a base is
// that it targets the ROOT. And because a tuple may omit the PathRef (arity
// disambiguates), `op[1]` is a PathRef or a value depending on length. These
// exist so nobody writes that comparison twice.
/** True for the `r` op. */
export declare function isReplace(op: Op): boolean;
/**
* True when this batch begins with `r` — a **base batch**. Exact rather than a
* heuristic, because flush guarantees `r` appears at index 0 or not at all.
* Storage tags such elements `"base"` so recovery can stop at the last one.
*/
export declare function isBase(ops: readonly (Op | WireOp)[]): boolean;
// ─── Codec ───────────────────────────────────────────────────────────────────
/**
* Path interning and arity omission. ONE PAIR PER STREAM — the id table spans a
* whole subscription or file, so a consumer joining later needs its own.
*
* The table resets on a base batch, because a reader replays from the last one
* with a fresh decoder and everything after it must be self-contained.
*/
export declare function encoder(): { encode(ops: readonly Op[]): WireOp[] };
export declare function decoder(): { decode(wire: readonly WireOp[]): Op[] };
// ─── Tracker ─────────────────────────────────────────────────────────────────
export interface TrackerOptions {
/**
* Cap on characters compared when deriving append/truncate from a string
* write. A window slide's overlap lies near the end, so a cap bounds cost
* without changing the result. Default 65536; 0 disables detection.
*/
maxOverlapScan?: number;
/** Merge compatible adjacent ops on flush. Default true. */
coalesce?: boolean;
}
export interface Tracker<T extends object> {
/**
* Mutate this. Plain TS: assignment, push, splice, delete, all of it.
*
* Assigning to it replaces the whole value: emits `["r", next]` and discards
* ops recorded before it. It must be a SETTER on the tracker — a plain
* property would be overwritten, swapping the proxy for a plain object and
* silently ending tracking.
*
* To force a base batch without changing the value, call `rebase()`.
*/
state: T;
/** The untracked backing object. Reads only — writing here emits nothing. */
readonly target: T;
/**
* Drain. Empty when nothing changed.
*
* The FIRST flush is always a base batch, carrying the value including any
* mutations made before it — a consumer starts with nothing, so a stream must
* open with something to apply deltas to.
*
* Ops that a later op makes unreachable are dropped first — a field written
* three times emits one op, and a parent replaced after its child emits one.
* Object key order is not preserved by that pass; see `delta.md` §5.
*/
flush(): Op[];
/** Discard without emitting. For a producer that just sent a base batch. */
discard(): void;
get dirty(): boolean;
}
export declare function track<T extends object>(root: T, options?: TrackerOptions): Tracker<T>;
// ─── Applier ─────────────────────────────────────────────────────────────────
/**
* Apply ops to a plain mutable object. Five verbs plus the dictionary entry, no
* domain knowledge, no library, no registry lookup.
*
* Returns the value, because a root set replaces it outright and cannot be done
* in place. Throws `PathError` when a path does not resolve, which a consumer
* treats exactly like a sequence gap: request a fresh base frame.
*
* Must not be pointed at a value produced by Immer, which deep-freezes.
*/
/**
* Fold ops into a value and return it — returned rather than mutated in place,
* because `r` replaces the value outright. Tolerates an `undefined` target,
* since a root op never reads it, so a consumer needs no base-batch branch.
*
* An `r` payload is **adopted, not copied**: the consumer owns the batch it was
* handed. Do not hand one batch to two in-process consumers, or their replicas
* will alias.
*/
export declare function apply<T>(target: T, ops: readonly Op[]): T;
export declare class PathError extends Error {
readonly path: Path;
}
// ─── Replica ─────────────────────────────────────────────────────────────────
export interface Replica<T> {
/** Current value. `undefined` until the first base frame. */
readonly value: T | undefined;
/**
* Feed one batch, with the sequence number the transport supplied. Returns
* "gap" when `seq` is not contiguous or a path failed to resolve; the caller
* then requests a fresh base batch.
*/
receive(ops: readonly Op[], seq: number): "ok" | "gap";
}
export declare function replica<T>(): Replica<T>;
// ─── Semantics ───────────────────────────────────────────────────────────────
//
// ROOT OPS
// `["s", [], v]` replaces the value; `["p", [], …]` splices it when the value
// is an array. The applier must special-case `path.length === 0` before
// resolving a parent — a prototype handling only a root set threw on the first
// randomised sequence that produced a root splice.
//
// STRINGS
// A write of `next` over `prev` derives ops from the longest suffix of `prev`
// that is a prefix of `next`:
// overlap === prev.length -> append(next.slice(overlap))
// overlap > 0 -> truncate(prev.length - overlap) + append(rest)
// overlap === 0 -> set(next)
// Always correct: the overlap is verified, not guessed. This is what makes a
// rolling window cheap — the case every effect-recording library degrades on.
//
// Use a native probe, not a hand-written KMP: `indexOf` a short head of `next`
// in `prev`, verify with `endsWith`. Measured 47x faster over 2000 slides on a
// 50 KB string, verified identical across 200.
//
// `t` counts UTF-16 code units, matching String.prototype.slice. Not bytes.
//
// ARRAYS
// Mutator methods are intercepted in the `get` trap, so intent is recorded
// before the engine performs index writes:
// push/unshift/pop/shift/splice -> one `p`
// sort/reverse/fill/copyWithin -> `s` of the whole array (rare)
// Direct index writes are honest `s` ops. `a.length = n` is translated —
// shrink to a splice, grow to a splice of nulls — never emitted as a path,
// because `length` is not a document location.
//
// TRANSACTIONS
// Several writes to one address in one commit must each be recorded against
// the previous one's result, not against pre-transaction state. The applier
// replays in order; anything else applies the second against a stale base.
//
// ALIASING
// Op payloads are structured-cloned at record time. Without this a later
// mutation retroactively changes an already-emitted op and a local replay
// aliases the source — an observed bug, not a hypothetical.
//
// UNDEFINED
// `x = undefined` normalises to `d`. JSON has no undefined, and a set with a
// missing value is indistinguishable from a lost value after a round trip.
//
// SCOPE
// JsonValue only. No Map, Set, Date, RegExp, class instances, getters, symbol
// keys, or cycles. That keeps this to five traps and one string algorithm, and
// makes structuredClone on payloads always safe.

604
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/02-scopes/implementation-handoff.md vendored
View File

@@ -1,604 +0,0 @@
# Scoped storage Step 1 — actionable implementation handoff
**Status: actionable, not implemented.**
**Prepared against:** `c4b0e35ab` (`dev`). Re-audit named source files if the implementation baseline changes materially.
This handoff implements storage scopes only. **Read this file before [`scopes.md`](scopes.md).** It supersedes conflicting details there, especially per-scope sequence spaces, operation-derived orphan detection, one-use scope IDs, raw-string scope arguments, sidecars ending in `.jsonl`, the proposed interned retirement tuple, and bundling JSONL delta/address encoding into the same change. The broader motivation and measurements in `scopes.md` remain design evidence; its signatures and implementation order are not current where this file differs.
Step 2—Chord-encoded JSONL values, address interning, compact tuple records, and their measurements—is a separate package. Do not begin Step 2 until Step 1 has been tested, reviewed, explicitly approved by the user, committed, and pushed.
## 0. Delivery protocol
This package has a mandatory user checkpoint:
1. Implement Step 1 only.
2. Run every required focused test, `npm run check`, and `./test.sh`.
3. Review the complete diff and report results. Delegated reviews, if used, run with provider `anthropic` and model `claude-fable-5`.
4. **Stop and wait for explicit user approval.**
5. After approval, commit only this package's files and push that commit.
6. Confirm the pushed commit before designing or implementing Step 2.
Do not combine the Step 1 and Step 2 commits. Do not commit or push before approval.
## 1. Mandatory reading
Read completely before editing:
1. `packages/agent/docs/harness.md` §§0.2–0.9, 1.1–1.7, 2.7–2.8, 3.7–3.8, 3.13, 4.4–4.8, 5.4, Part 7, Part 9.
2. `packages/agent/docs/values.md`.
3. `packages/agent/docs/mobile-handoff/README.md`.
4. [`scopes.md`](scopes.md), using this handoff where they conflict.
5. `../01-delta/delta.md` for vocabulary ownership only; Step 1 does not store Chord ops.
6. `../04-tool-output/harness-tools.md` §§7.1–7.7 for later-consumer context; do not implement its output redesign here.
7. `../05-assistant-output/message-update.md` §7 for later-consumer context; do not implement it here.
8. `packages/agent/src/harness/session/{types,values,commit,in-memory-storage-state,memory,session,index}.ts`.
9. Every file under `packages/agent/src/harness/session/jsonl/` and the `FileSystem` declaration/implementation in `packages/agent/src/harness/{types,env/nodejs}.ts`.
10. `packages/agent/src/harness/runtime/{progress,lane}.ts`, `runtime/drive/{terminal,response,deferred,tools,reconcile,structural,boundary}.ts`, and every caller of `operationCleanupWrites`. Inspect `tool-placement.ts` through the grep audit, but current source has no scoped delete there.
11. `packages/session-backends/sqlite-node/src/{index,sqlite/index}.ts`, `sqlite/{storage,repo}.ts`, `migrations/001_initial.sql`, and `session/{values,session-sequences}.ts`.
12. Shared storage/repository conformance and every focused test named in §9.
13. `scopes.variance.ts` for the covariant-address/invariant-write phantom mechanism only. Its current no-ID `EphemeralScope` and `retireScope(id: string)` signatures are superseded by §2 and must be updated in Slice A.
Do not use `dist/` as implementation input. Format 4 remains WIP; no R11 migration is part of this package.
## 2. Fixed scope contract
### 2.1 Runtime scope identity
A scope ID is a reusable logical lifetime name, not a globally one-use token:
```ts
export type SessionScope = { readonly kind: "session" };
export interface EphemeralScope {
readonly kind: "ephemeral";
readonly id: string;
}
export type Scope = SessionScope | EphemeralScope;
export function ephemeralScope(id: string): EphemeralScope;
```
`ephemeralScope(id)` requires a non-empty well-formed Unicode string and returns a frozen value. Equal IDs identify the same physical scope; object identity has no meaning. JSONL encodes IDs with `encodeURIComponent`. The encoded component must be at most 180 ASCII characters; reject longer IDs and lone-surrogate input. The physical filename always has a fixed `scope-` prefix, so encoded `.`/`..` values cannot become path segments. Do not directly interpolate unencoded caller input into a path.
No create/open-scope transaction exists. The first scoped value/list write creates physical state. A later `retireScope(scope)` ends everything in that scope through its assigned global sequence. A write under the same ID after retirement begins a new logical lifetime.
Harness operation scopes use `ephemeralScope(operationId)`. Generic application scopes may use stable names. Storage never parses a scope ID as an operation ID and never reads Harness operation state to decide lifetime.
### 2.2 Reuse and retirement boundary
Global sequence order defines reusable generations:
```text
seq 10 scoped write S
seq 20 scoped write S
seq 30 retireScope(S)
seq 40 scoped write S # new lifetime
```
For a given scope ID, only scoped records with `seq` greater than its latest committed retirement sequence are live. Memory and SQLite realize this by deleting current rows/maps at retirement and allowing later writes to recreate them. JSONL uses the retirement sequence as its replay boundary (§6).
Retirement is idempotent in the storage sense: retiring a scope with no current state is legal and advances the boundary. A later write still begins a new lifetime. Storage does not maintain an all-time used-ID registry or reject reuse.
A scoped write reaching the commit line after retirement is therefore a new-lifetime write. Harness invocation fencing and settlement drains must continue to ensure a late assistant/tool progress job cannot recreate operation output after terminal retirement. Add explicit tests for this boundary.
### 2.3 Explicit retirement is the sole authority
```ts
export function retireScope(scope: EphemeralScope): Write<SessionScope>;
```
Storage never infers orphanage from operation absence, namespace, current state, or a missing owner. The only valid lifetime boundary is a committed `retireScope` write. If an owner never retires its scope, that is an owner defect and the scope remains live.
Crash behavior:
- crash before the retirement transaction commits: the scope remains live and reopens;
- crash after retirement commits but before JSONL unlink: replay sees the retirement boundary, ignores pre-boundary sidecar records, and retries physical deletion;
- crash during a later reused lifetime: records after the latest retirement reopen normally.
Repository deletion separately removes every physical sidecar belonging to the deleted Session. It does not need operation semantics.
### 2.4 Address and write typing
Addresses carry a covariant scope tag; writes carry an invariant scope tag. Preserve value-type invariance independently:
```ts
interface Value<T, Sc extends Scope = SessionScope> { /* existing fields + scope */ }
interface ValueList<T, Sc extends Scope = SessionScope> { /* existing fields + scope */ }
declare function value<T>(namespace: string, key?: string): Value<T, SessionScope>;
declare function value<T>(namespace: string, key: string, scope: EphemeralScope): Value<T, EphemeralScope>;
declare function list<T>(namespace: string, key?: string): ValueList<T, SessionScope>;
declare function list<T>(namespace: string, key: string, scope: EphemeralScope): ValueList<T, EphemeralScope>;
```
A session-scoped address has no runtime scope ID. An ephemeral address carries its scope ID. Scope is part of physical identity: equal namespace/key addresses in session scope and ephemeral scope, or in two different ephemeral IDs, are distinct.
`setValue`, `deleteValue`, `appendList`, and `deleteList` preserve the address scope in `Write<Sc>`. Entry and usage writes are session-scoped. `retireScope` is session-scoped even though applying it deletes ephemeral state.
All writes in an ordinary transaction have one static scope. A session transaction may include session values, entries, usage, and one or more `retireScope` writes. It may not include a direct ephemeral set/delete/append. An ephemeral transaction may include value/list writes only. At runtime, every ephemeral write in one transaction must carry the same scope ID; two different IDs have the same TypeScript tag and require this assertion.
Make the minimum generic changes necessary to preserve this through `Write`, `CommittedWrite`, `Storage.commit`, Session mutation capabilities, `CommitDecision`, and lane commands. Follow the variance proof in `scopes.variance.ts`; do not make readers invariant or propagate unnecessary scope type parameters through read-only APIs.
### 2.5 One global sequence space
Memory, JSONL main records, JSONL sidecar records, and SQLite rows share one Session-global sequence space. Every admitted commit remains serialized and receives increasing sequences in admission order. Gaps remain legal.
- Memory continues using one `nextSeq`.
- SQLite continues allocating from `sessions.next_seq` inside the write transaction.
- JSONL main and sidecar appends share one commit queue and one resident `nextSeq`.
JSONL open computes the high-water mark from the main header and every complete record in every matching sidecar **before** deleting or ignoring retired physical files. A retired sidecar's sequences may then become gaps but can never be reused. Torn uncommitted final transactions do not advance the durable high-water mark.
No reservation record, per-scope sequence counter, or range allocator is permitted.
## 3. List tags
Step 1 adds only the mechanism required by later tracked-output recovery:
```ts
export interface ListElement<T> {
seq: number;
value: T;
tag?: string;
}
export interface ListReadOptions {
cursor?: ListCursor;
order?: "asc" | "desc";
limit?: number;
stopAtTag?: string;
}
export function appendList<T, Sc extends Scope>(
address: ValueList<T, Sc>,
element: NoInfer<T>,
tag?: string,
): ListAppendWrite<Sc>;
```
Tags are non-empty strings when present. Storage stores and returns the tag without interpreting the element.
Read semantics:
1. Apply address scope, exclusive cursor, and order.
2. Inspect at most the normalized `limit` elements.
3. Stop at and include the first element carrying `stopAtTag`.
4. Return that possibly shortened page.
`stopAtTag` never searches beyond the page limit. If absent from the page, the caller pages again from the last returned sequence. Do **not** add the draft `tag` filter; no current consumer needs it and combining filtering with the existing cursor shape is ambiguous.
SQLite may fetch the indexed `limit` rows and truncate at the first returned tag in TypeScript; storage still never parses payloads. Preserve the existing primary-key query plan with no temporary sort.
## 4. Step 1 durable payloads remain unchanged
This package changes lifetime and routing, not output representation:
- `pendingToolOutput` remains `Value<AgentToolResult<unknown>, EphemeralScope>`;
- `pendingAssistantFrames` remains `ValueList<AssistantMessageFrame, EphemeralScope>`;
- `operationToolMemo` remains a `JsonValue` scalar;
- JSONL transaction records remain readable keyed objects;
- assistant progress still appends one frame per accepted frame;
- tool checkpoints still replace whole snapshots;
- `message_update` and `tool_update` event shapes do not change;
- safe replay retains its current checkpoint behavior, including the known delete-before-replay bug owned by the later tool-output package.
Do not rename `pendingAssistantFrames`, introduce `pendingAssistantOutput`, store `WireOp[]`, add Chord trackers/codecs, redesign `AgentHarnessTool`, or change output cadence here.
## 5. Runtime ownership and cleanup
### 5.1 Addresses moved to the operation scope
Construct these addresses with `ephemeralScope(operationId)`:
- `operationToolMemo(operationId, invocationId, name)`;
- `pendingToolOutput(operationId, invocationId)`;
- `pendingAssistantFrames(operationId, responseEntryId)`.
All other current built-ins remain session-scoped. In particular, `pi.op.state`, `pi.op.meta`, `pi.op.tool_args`, `pi.op.preparation`, and `pi.pending.entry` remain in the main scope because their writes coordinate atomically with lane/operation state.
### 5.2 No cross-file cleanup transaction
Remove direct ephemeral deletes from transactions that also write session state:
- assistant response settlement;
- deferred-response supersession;
- tool outcome staging in `drive/tools.ts`;
- cancellation/recovery decisions;
- terminal per-address cleanup.
Current `drive/tool-placement.ts` has no ephemeral delete; do not invent a change there.
`operationCleanupWrites` stops scanning tool memos and tool outputs and stops constructing a state-directed assistant-frame delete. It keeps session-scoped operation/meta/args/preparation/pending-entry cleanup and adds exactly one `retireScope(ephemeralScope(operationId))` in the universal terminal suffix.
After removing the current staging/terminal scans, remove `operationToolMemoPrefix` and `pendingToolOutputPrefix`; current source has no other production consumers. Update the normative exported prefix-constructor count from five to three and update exact constructor tests rather than retaining dead inventory APIs.
An ephemeral-only delete remains legal. For example, the current safe-replay checkpoint delete remains because its transaction contains no session write; fixing its behavior belongs to the later tool-output package, not scoped storage or JSONL optimization.
### 5.3 Unreachable scoped residue
After a response settles, a deferred response is superseded, or a tool reaches `outcome_ready`, its former frames/checkpoints/memos may remain physically and logically addressable inside the still-active operation scope. Current scalar operation state no longer references them; no recovery or snapshot path scans the scope to infer authority. The terminal retirement deletes the whole scope.
Call this **unreachable scoped residue**, not orphanage. Update `harness.md` invariants that currently require an `outcome_ready` call to have no physically present memo/checkpoint. The replacement invariant is that settled child state never consumes or exposes residue and terminal retirement removes all scoped state atomically with operation completion.
Close and fault are controlled crashes and never retire the scope. Reopen restores scoped progress only through addresses derived from current authoritative operation state.
## 6. JSONL sidecars
### 6.1 Physical layout
The main session file remains unchanged except for committed scope-retirement records. Every main file owns one known sidecar directory at `${mainPath}.scopes`; this requires no dirname API or `FileSystem` extension. Create it lazily with `createDir(..., { recursive: true })`. Discover scopes with `listDir` on that exact path. A sidecar filename is exactly `scope-${encodeURIComponent(scope.id)}.scope`, subject to §2.1's 180-character encoded-component limit. The suffix does not end in `.jsonl`, and the sidecar directory itself is a directory, so repository listing never mistakes either for a Session.
A sidecar begins with this exact header:
```ts
interface JsonlScopeHeader {
v: 4;
kind: "scope_header";
sessionId: string;
scopeId: string;
storageVersion: 1;
}
```
Validate the exact session/scope identity before replay. Create the first header through a temp file inside `${mainPath}.scopes` plus atomic rename. Temporary filenames end in `.tmp` and are never discovered as sidecars. Use the existing `FileSystem` capability; do not add a dirname method, extend `JsonlStorageOptions`, or add Node-only filesystem access to agent core.
Step 1 sidecar writes retain the current object record spelling and carry/validate their scope ID in committed shapes. Each physical transaction remains one complete line or one array line. Sidecar transactions may contain only value/list writes for that exact scope.
### 6.2 Retirement record
The main log records an explicit committed write:
```ts
interface CommittedScopeRetireWrite {
kind: "scope";
op: "retire";
seq: number;
scopeId: string;
}
```
Its Step 1 JSON representation is the corresponding readable object. Do not use the draft `['!', addrId, seq]`: a scope is not an interned value/list address. Compact tuple spelling belongs to Step 2 and must be designed there.
Applying a committed retirement removes resident values/lists in that scope. After the complete main transaction is durable, JSONL synchronously serializes lifecycle cleanup on the same commit queue: close any owned sidecar resource, attempt unlink, then release the queue. Failure to unlink is non-fatal to the already-committed business transaction; the sidecar remains logically bounded by the retirement sequence and deletion is retried on reopen.
### 6.3 Replay and reusable IDs
Open performs these logical stages before admitting writes:
1. Read/repair the main file and parse complete main transactions.
2. Discover and validate matching sidecars; ignore unrelated files.
3. Parse complete sidecar transactions and repair torn final transactions.
4. Compute the global sequence high-water mark from the header and **all** complete main/sidecar records, including records later excluded by retirement.
5. Determine the latest retirement sequence per scope from main transactions.
6. Keep sidecar writes whose sequence is greater than that scope's latest retirement.
7. Merge retained main and sidecar transactions by global sequence and replay them in global order, preserving transaction boundaries and rejecting duplicate/non-monotonic sequences.
8. Advance to the precomputed high-water mark.
9. Remove sidecars with no records after their latest retirement; cleanup failure does not make retired contents live.
Main transactions have sequence gaps where sidecar transactions occurred. Sidecars have gaps where other scopes/main commits occurred. Both remain valid.
A sidecar containing records after a retirement is an active reused lifetime and must not be deleted. Serialize retirement cleanup against later writes so an old unlink cannot race and remove a newly reused sidecar.
Malformed interior records or identity mismatch in an active sidecar are storage corruption and fail open. A torn final transaction is discarded wholly. Temporary files left by interrupted atomic creation are cleanup artifacts and never become sidecars.
### 6.4 Legacy v3
Legacy v3 has no scopes. An ordinary Harness operation commits session-scoped acceptance before any progress write, so it upgrades first in the normal path. Still define and test the generic storage behavior when the first caller write is ephemeral: complete the existing v3-to-v4 main rewrite/usage adjustment under the global commit queue, then commit the scoped transaction without placing it in the main file. A crash before the sidecar append leaves no committed caller transaction and may reuse its uncommitted sequence.
### 6.5 Repository and fork behavior
- Repository `list()` ignores `${mainPath}.scopes` directories and their files through its existing non-file/`.jsonl` filtering.
- Repository `delete()` removes the main file through its current path, then removes `${mainPath}.scopes` recursively with `force: true`; a scope-directory cleanup failure rejects deletion after the main-file result is known, without inferring Harness state. Missing scope directories are legal.
- `JsonlStorage.close()` drains its admitted main/sidecar commits as today; `JsonlSessionRepo.close()` retains its current lifecycle behavior because repository ownership is a separate package. No progress scope is retired merely because a handle closes.
- Preserve the current fork allowlist exactly. Do not add generic application values or lists to either fork scope. Ensure source snapshots and backend fork readers ignore all ephemeral values/lists regardless of namespace; current destinations remain logically unchanged except that ephemeral state cannot leak.
- Fork destinations contain no sidecars or retirement state.
- WP08 may later replace fork materialization but must preserve this scope policy.
J1 compaction is excluded. When J1 eventually lands, it may omit a historical retirement record only after no physical sidecar can be made live by losing that boundary.
## 7. Memory backend
Extend `InMemoryStorageState` physical value/list identity with scope. Session scope and each ephemeral ID are distinct. Preserve one global `nextSeq` and current stats behavior.
Applying `CommittedScopeRetireWrite` deletes every scalar/list in that scope in the same synchronous application of the surrounding transaction. It does not delete entries or usage, which cannot be ephemeral. A later scoped write recreates state under the same ID.
Snapshots expose scope so fork code can reject ephemeral state, but `createForkSnapshot` keeps its current built-in allowlist and must not start copying generic session-scoped application values or lists. Instrumentation must still report exact committed write order, including retirement.
## 8. SQLite backend
Change WIP schema in place with no migration:
```sql
scalar_values(
session_id TEXT NOT NULL,
scope_id TEXT NOT NULL,
namespace TEXT NOT NULL,
key TEXT NOT NULL,
seq INTEGER NOT NULL,
value TEXT NOT NULL,
PRIMARY KEY(session_id, scope_id, namespace, key)
) WITHOUT ROWID;
list_values(
session_id TEXT NOT NULL,
scope_id TEXT NOT NULL,
namespace TEXT NOT NULL,
key TEXT NOT NULL,
seq INTEGER NOT NULL,
value TEXT NOT NULL,
tag TEXT,
PRIMARY KEY(session_id, scope_id, namespace, key, seq)
) WITHOUT ROWID;
```
Use `scope_id = ''` for session scope and the exact scope ID otherwise. Every point read, prefix scan, set/delete, append/delete, snapshot, and query-plan assertion includes scope identity.
Inside the same `BEGIN IMMEDIATE` transaction, a retirement write executes:
```sql
DELETE FROM scalar_values WHERE session_id = ? AND scope_id = ?;
DELETE FROM list_values WHERE session_id = ? AND scope_id = ?;
```
It still consumes its assigned global sequence through `sessions.next_seq`, even though no retired-scope tombstone row remains. A later write under that ID recreates rows naturally. Retirement rollback must restore both scoped rows and all sibling main writes.
Fork paths copy only `scope_id = ''`. Shared-container deletion remains Session-scoped and removes all values/lists regardless of scope along with the other Session rows.
## 9. Required tests
Port tests before or with each implementation slice. Use the repository Vitest binary from the owning package. Do not run the full Vitest suite directly.
### 9.1 Type and constructor tests
- Session addresses default to `SessionScope`.
- Ephemeral overloads retain `EphemeralScope` while preserving invariant `T`.
- Address scope is covariant for reads; write scope is invariant.
- Same-scope session transactions typecheck.
- Same-scope ephemeral transactions typecheck.
- Session writes plus `retireScope` typecheck.
- Direct session + ephemeral writes in one transaction fail with `@ts-expect-error`.
- Two ephemeral IDs evade static distinction but fail the runtime same-ID assertion.
- Scope object identity is irrelevant; equal IDs access the same state.
- Empty and overlong encoded IDs reject; separators/Unicode encode without path escape.
- Session and ephemeral addresses with equal namespace/key do not alias.
### 9.2 Shared backend conformance
Run identically on Memory, JSONL, and SQLite:
- ephemeral scalar set/get/replace/delete;
- ephemeral list append/page/whole-list-delete;
- independent scopes with equal namespace/key;
- session scope independent from equal ephemeral address;
- mixed session/ephemeral transaction rejection before mutation;
- two ephemeral IDs in one transaction rejection before mutation;
- retirement deletes every value/list in the exact scope and no other scope;
- retirement atomic with session sibling writes;
- retirement of absent state is legal;
- write after retirement recreates a new lifetime;
- repeated retirement advances the boundary without making later writes disappear;
- main, scope A, scope B, retirement, and reused-A commits receive globally increasing sequences in admission order;
- retired/deleted physical elements leave legal gaps and first post-reopen commit allocates above every complete prior record;
- list tags round-trip, page limits remain bounded, and ascending/descending `stopAtTag` includes the marker;
- stats ignore scoped values/lists and retirement;
- close drains admitted scoped commits and rejects later reads/writes.
### 9.3 JSONL focused tests
- first scoped write creates exactly one valid sidecar and no main value/list record;
- several writes in one scoped transaction remain one physical sidecar line;
- main and multiple sidecars replay in global sequence order;
- header `nextSeq` and all complete sidecar records contribute to reopen high-water;
- torn sidecar final transaction is discarded wholly and repaired;
- malformed interior active sidecar and header identity mismatch fail open;
- crash before retirement leaves scoped data live;
- crash after main retirement but before unlink leaves data logically absent and cleanup retries;
- unlink failure does not reject the already-committed terminal transaction;
- reuse after retirement replays only records after the latest retirement sequence;
- multiple retirement/reuse cycles choose the latest boundary;
- retirement cleanup is serialized so a later reuse cannot be deleted by an earlier unlink;
- ignored pre-boundary records still prevent sequence reuse;
- sidecars and temp files never appear in repository listing;
- repository deletion removes all sidecars;
- forks exclude generic application-owned ephemeral values/lists;
- legacy-v3 first session write and first scoped-write paths both produce valid v4 state;
- close/reopen preserves active operation progress and does not retire it.
### 9.4 SQLite focused tests
- schema and query helpers include `scope_id`; list rows preserve nullable tags;
- one transaction atomically deletes scoped scalar/list rows and writes terminal main state;
- forced failure after scope deletion rolls back deletion and sibling writes;
- reuse after retirement recreates rows under the same scope ID;
- `sessions.next_seq` advances for retirement;
- list paging plan still uses the scoped primary key with no temp b-tree;
- both per-file and shared-container layouts isolate Session and scope IDs;
- fork and Session deletion exclude/remove scoped rows correctly.
### 9.5 Harness integration
Update existing exact-write tests rather than weakening them:
- assistant/deferred settlement drops mixed-scope frame deletes;
- tool outcome staging drops mixed-scope checkpoint/memo deletes;
- unreachable residue is never consumed by snapshot/recovery after its owning child settles;
- safe-replay's current ephemeral-only delete remains ordered and fenced;
- every terminal leaf emits one retirement in the documented cleanup position;
- close/fault emits no retirement and reopen recovers current frames/checkpoint/memos;
- cancellation and unknown-outcome recovery retire only at terminal completion;
- late queued assistant/tool progress cannot commit after terminal retirement or reopen the scope;
- terminal cleanup no longer scans memo/output prefixes;
- every focused test and fixture found by the §12 old-delete/prefix greps is updated deliberately; do not rely on a historical numeric test count;
- every terminal path reached through `operationCleanupWrites` callers in `drive/{response,reconcile,structural,boundary}.ts` is covered, including each operation family and cancellation.
Likely focused files include:
```text
packages/agent/test/harness/values.test.ts
packages/agent/test/harness/{memory,jsonl}-storage*.test.ts
packages/agent/test/harness/{memory,jsonl}-session-repo*.test.ts
packages/agent/test/harness/runtime/drive-{terminal,retry-deferred,tools,reconcile,generation}.test.ts
packages/session-backends/sqlite-node/test/{storage,storage-conformance,repo,repo-conformance}.test.ts
```
Use compiler and grep guards to find additional affected tests rather than assuming this list is exhaustive.
## 10. Implementation slices
### Slice A — contract, types, Memory reference, list tags
Primary files:
```text
CREATE packages/agent/src/harness/session/scope.ts for `SessionScope`, `EphemeralScope`, `Scope`, `ephemeralScope`, scope phantoms/helpers, and runtime scope-ID validation
MODIFY packages/agent/src/harness/session/{types,values,commit,in-memory-storage-state,memory,session,index}.ts
MODIFY packages/agent/src/harness/session/testing/{conformance/storage,instrumented-storage,storage-decorator,gating-storage}.ts
MODIFY packages/agent/test/harness/{values,memory-conformance,memory-storage,storage-backed-session}.test.ts
MODIFY packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.variance.ts
```
1. Add scope values/address/write variance and reusable-ID semantics.
2. Add retirement committed write and one-scope transaction validation.
3. Add tag/`stopAtTag` behavior.
4. Implement Memory physical identity, retirement, global sequences, snapshots.
5. Pass type tests and Memory conformance before continuing.
### Slice B — SQLite
Primary files:
```text
MODIFY packages/session-backends/sqlite-node/src/sqlite/migrations/001_initial.sql
MODIFY packages/session-backends/sqlite-node/src/sqlite/{storage,repo}.ts
MODIFY packages/session-backends/sqlite-node/src/sqlite/session/{values,session-sequences}.ts
MODIFY packages/session-backends/sqlite-node/test/{storage,storage-conformance,repo,repo-conformance}.test.ts
```
1. Add scoped schema/query identity and tags.
2. Implement atomic `DELETE ... WHERE scope_id` retirement.
3. Preserve global sequence allocation, stats, branch indexes, shared-container behavior.
4. Exclude ephemeral rows from forks.
5. Pass SQLite focused/conformance tests before continuing.
### Slice C — JSONL sidecars
Primary files:
```text
CREATE packages/agent/src/harness/session/jsonl/scope-files.ts for fixed directory/file naming, exact header parsing/serialization, discovery, atomic first creation, torn-tail parsing/repair, and recursive cleanup helpers
MODIFY packages/agent/src/harness/session/jsonl/{types,codec,storage,repo,legacy-v3,index}.ts
MODIFY packages/agent/test/harness/jsonl-{storage,storage-conformance,session-repo,session-repo-conformance,v3-migration}.test.ts
```
1. Add sidecar naming/header/create/discovery.
2. Route scoped commits to sidecars while allocating globally.
3. Persist main retirement records and serialized unlink.
4. Implement multi-file high-water calculation and globally ordered replay with reusable boundaries.
5. Handle torn tails, corruption, v3, listing, delete, close, and forks.
6. Pass all JSONL tests before runtime migration.
### Slice D — Harness migration
Primary files:
```text
MODIFY packages/agent/src/harness/session/values.ts
MODIFY packages/agent/src/harness/runtime/{progress,lane}.ts to propagate scoped write generics through progress channels and lane commands
MODIFY packages/agent/src/harness/runtime/drive/{terminal,response,deferred,tools,reconcile,structural,boundary}.ts
INSPECT packages/agent/src/harness/runtime/drive/tool-placement.ts; current source needs no scoped cleanup change
MODIFY all focused runtime tests found by the §12 greps
```
1. Scope memo/tool/frame addresses by operation ID.
2. Remove mixed-scope deletes and cleanup scans.
3. Add terminal retirement in the universal suffix.
4. Preserve current output/replay/event behavior.
5. Prove late-write fencing and every terminal/close/recovery path.
### Slice E — documentation and final validation
Update current normative/reference docs:
```text
packages/agent/docs/harness.md
packages/agent/docs/values.md
packages/agent/docs/post-wp05-roadmap.md
packages/agent/docs/mobile-handoff/README.md
packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.md
packages/session-backends/sqlite-node/README.md
packages/agent/src/harness/telemetry.ts and generated `packages/agent/docs/telemetry-schema.md`: add the `scope` session-write item kind without implementing spans
```
Do not rewrite historical WP00–WP07 handoffs or released changelog sections. On `dev`, do not add changelog entries under the repository's main-only rule.
Run:
```bash
npm run check
./test.sh
```
Also run every modified focused test after each slice. Record full command results for the approval checkpoint.
## 11. Exclusions
Do not include:
- Chord `Op`/`WireOp` storage integration;
- JSONL address/path interning or compact tuple records;
- pending-output address renames;
- `ToolOutput`, tool API changes, replay seeding, memo/checkpoint atomicity, progress cadence, rate limiting, exec-env changes;
- compact `message_update`, assistant incremental reducer changes, or protocol replication changes;
- J1 main-log snapshot compaction;
- WP08 fork redesign beyond excluding ephemeral state from current forks;
- repository lifecycle contract changes;
- telemetry implementation beyond keeping the declared schema accurate for the new write kind;
- migrations or compatibility for older WIP format-4 SQLite schemas;
- application-level automatic scope ownership, operation-derived orphan inference, leases, TTLs, or background scope collection.
If implementation requires an excluded item, stop and revise this handoff before expanding scope.
## 12. Grep guards
Before final review, inspect every remaining match:
```text
operationToolMemoPrefix
pendingToolOutputPrefix
deleteList(pendingAssistantFrames
deleteValue(pendingToolOutput
scopeId
retireScope
stopAtTag
```
Expected outcomes:
- no session-scoped transaction directly deletes an ephemeral address;
- no terminal path omits retirement;
- no backend point read/write omits scope identity;
- no fork copies ephemeral state;
- no JSONL sidecar is treated as a Session file;
- no per-scope sequence counter exists;
- no operation-state/orphan inference exists in storage.
## 13. Exit condition
Step 1 is implementation-complete when:
- scope/address/write typing rejects mixed-scope transactions and runtime validation rejects two ephemeral IDs in one transaction;
- IDs are reusable through latest-retirement global sequence boundaries with no historical ID registry;
- Memory, JSONL, and SQLite share one global sequence space and pass common conformance;
- SQLite retirement deletes exact scoped rows atomically with terminal state;
- JSONL writes ephemeral values/lists only to sidecars, records retirement only in the main log, replays all files in global order, preserves high-water marks, and deletes retired sidecars without operation inference;
- list tags and bounded `stopAtTag` work identically across backends;
- current tool/assistant payload and event formats are unchanged;
- every terminal path retires once, while close/fault preserves active scope recovery;
- forks and repository listing/deletion handle sidecars correctly;
- focused tests, `npm run check`, and `./test.sh` pass;
- final review has no blocker;
- the complete diff and results have been presented to the user and implementation has stopped for approval.
Only after explicit approval may Step 1 be committed and pushed. Step 2 starts only after that push is confirmed.

574
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.md vendored
View File

@@ -1,574 +0,0 @@
# Session Storage: Scopes
> **Status:** Design evidence. The actionable Step 1 contract is [`implementation-handoff.md`](implementation-handoff.md), which supersedes this document on sequence allocation, reusable scope IDs, retirement authority/records, sidecar layout, implementation order, and the separation of JSONL encoding into Step 2.
>
> **Scope:** `packages/agent/src/harness/session/`, plus the call sites in
> `runtime/` listed in §10.
>
> **Depends on [01-delta](../01-delta/delta.md)** for the landed Chord op vocabulary that pending state is stored in (§2, §11) and for the `WireOp` form the record encoding carries (§12). The two
> are otherwise independent and compound: encoding shrinks every write, scopes
> move one class of write out of the main log entirely.
>
> Every claim here was checked against `runtime/lane.ts`, `runtime/progress.ts`
> and `runtime/drive/*.ts`. The type-level enforcement in §6 was verified with
> `tsc --strict`, including that mixed-scope commits are the *only* thing it
> rejects.
## 1. The problem
Operation state — pending tool output, pending assistant output — matters only
while the operation runs. It is crash-recovery scaffolding, not history.
The JSONL log is append-only, so it persists anyway. Worse, deleting it *adds*
records: `deleteValue` and `deleteList` write a line each, and those lines also
persist. Today only a full snapshot rewrite reclaims the space, and no in-place
compaction exists (`jsonl/storage.ts` has `createFromSnapshot`, used for fork).
Measured on 20 operations, each 400 assistant frames plus 2 tools at 60
checkpoints of a 50 KB window:
| | size |
| --- | --- |
| total JSONL | 93.89 MB |
| settled transcript entries — the part that is actually history | **0.06 MB** |
Three orders of magnitude of the file is scaffolding for finished operations.
**Two independent fixes, and they are not alternatives:**
- **Encoding** ([delta.md](../01-delta/delta.md)): value writes carry Chord ops rather than whole values, and addresses are interned. Takes the same workload to **5.32 MB in a single file**, with no change to atomicity. The op vocabulary and flush-time tracker have landed; storage integration has not.
- **Scopes** (this document): pending state leaves the main log entirely, so it is
never history to begin with.
Do the encoding first. Scopes are worth it because 5.32 MB of dead weight per 20
operations still accumulates, and there is no compaction pass to reclaim it.
## 2. Scope on the address
Scope has two halves that must not be confused: a **type-level tag** carrying no
data, and a **runtime scope id** that names the file.
```ts
export type SessionScope = { readonly kind: "session" };
export type EphemeralScope = { readonly kind: "ephemeral" };
export type Scope = SessionScope | EphemeralScope;
// Passing a scopeId is what makes an address ephemeral.
export function value<T>(namespace: string, key: string): Value<T, SessionScope>;
export function value<T>(namespace: string, key: string, scopeId: string): Value<T, EphemeralScope>;
export function list<T>(namespace: string, key: string): ValueList<T, SessionScope>;
export function list<T>(namespace: string, key: string, scopeId: string): ValueList<T, EphemeralScope>;
/** A main-log record retiring an ephemeral scope. See §5. */
export function retireScope(id: string): Write<SessionScope>;
```
The suffix is not decoration: `Session` already names the session interface in
`harness/session/types.ts`, and reusing it produces `TS2440`/`TS2484` plus an
ambiguous re-export from `session/index.ts`.
```ts
interface Value<T, Sc extends Scope = SessionScope> extends ScopedAddress<Sc> {
readonly namespace: string;
readonly key: string;
/** Present iff Sc is Ephemeral. Routes the write and names the sidecar. */
readonly scopeId?: string;
}
```
The id is a plain runtime string, because it is an operation id — generated at
runtime and unavailable to the type system. That is precisely why §6 can
distinguish *session from ephemeral* statically but cannot distinguish *two
different ephemeral scopes*: the tag is in the type, the id is not.
```ts
// Durable lists carry encoded batches; one encoder/decoder pair belongs to each value stream.
export const pendingToolOutput = (operationId: string, invocationId: string) =>
list<WireOp[]>("pi.pending.tool_output", `${operationId}:${invocationId}`, operationId);
export const pendingAssistantOutput = (operationId: string, entryId: string) =>
list<WireOp[]>("pi.pending.assistant_output", `${operationId}:${entryId}`, operationId);
export const operationToolMemo = (operationId: string, invocationId: string, name: string) =>
value<JsonValue>("pi.op.tool_memo", `${operationId}:${invocationId}:${name}`, operationId);
// unchanged — no scopeId, so Session by inference
export const laneStateValue = (lane: string) => value<DurableLaneState>("pi.lane.state", lane);
```
**Scope is the file.** A session-scoped write goes to the main log; an
ephemeral-scoped write goes to `<session>.<scopeId>.jsonl`.
Other backends need no sidecars at all: in-memory drops a `Map` keyed by scope id,
SQLite issues one `DELETE ... WHERE scope = ?`. Scope is meaningful to them only as
a lifetime hint. It is JSONL that needs the file split, and that is where the
constraints in §4 and §5 come from.
## 3. Which addresses are scoped, and why
The boundary is drawn by **write isolation**, not by lifetime.
Scoping everything under `pi.op.*` on the grounds that it dies with the operation
looks right and is wrong. `lane.ts` shows why: **every** operation commit bundles
`operationState` with `laneState`.
```ts
// lane.ts:405/431, 659/660, 749/750, 1071/1072 — the same shape each time
writes: [
...decision.writes,
setValue(operationStateValue(operationId), decision.operationState),
setValue(laneStateValue(this.name), durableLaneState(...)),
]
```
That pair is coordination state: lane state says the operation is at step N,
operation state holds the data for step N. Splitting them across files makes a
crash produce a lane that believes it is somewhere the data does not support.
The **bulk** values behave differently. `openProgress` commits exactly one write
(`runtime/progress.ts:51`), and `pendingToolOutput` / `pendingAssistantFrames`
appear in multi-write transactions only as *deletes*.
| | addresses |
| --- | --- |
| **ephemeral (sidecar)** | `pi.pending.tool_output`, `pi.pending.assistant_output`, `pi.op.tool_memo` |
| **session (main log)** | `pi.lane.state`, `pi.op.state`, `pi.op.meta`, `pi.result`, `pi.pending.entry`, `pi.branch.tip`, `pi.op.tool_args`, `pi.op.preparation` |
`operationToolMemo` is ephemeral so that the memo-plus-checkpoint bundling in
`harness-tools.md` §7.5 is a single-file transaction. `terminal.ts:34` already
treats `operationToolMemoPrefix` and `pendingToolOutputPrefix` as the same cleanup
class, so this matches how the code already thinks about them.
The size win is unaffected by leaving `operationState` behind: effectively all of
the 93.89 MB in §1 is pending tool and assistant output.
## 4. Two files are not atomic
Serialising on the Session line gives **ordering, not atomicity**. Two `write()`
calls to two descriptors, two fsyncs, no journal spanning them. A crash between
leaves the files disagreeing, and JSONL has nothing to repair it with.
So the design does not try to make cross-file transactions work. It ensures none
exist:
> **All writes in a transaction share one scope.**
That is achievable because it is already true of the code — see §5 for the audit.
Two consequences of *not* needing cross-file commits:
- **No write ordering between files is required.** The commit-record pattern
(sidecar first, fsync, then a main-log record acknowledging it) is unnecessary if
no transaction spans files. There is nothing to acknowledge.
- **No commit marker for sidecar-only writes.** One tempting design is a small
main-log record for `openProgress`, since nothing acknowledges its write. It is
not needed: that transaction is one write in one file and already atomic. Torn
tails are handled the same way as in the main log — every line is a complete op
or a whole batch, so a reader stops at the last intact one.
What remains is a **durability** choice, not a consistency one: if the sidecar is
fsynced less aggressively, recent checkpoints can be lost. That is the same bounded
loss the checkpoint interval already accepts, but it should be a deliberate policy
rather than an accident.
## 5. Audit: does the existing code satisfy the rule?
Every commit in `lane.ts` (12 sites) and every `writes:` producer in
`runtime/drive/*.ts` (32 sites) was read.
**Result: yes, with one adjustment.** No transaction writes to two files. The
spanning cases are all *deletes* of ephemeral state bundled with main-log writes:
- `response.ts:340` — the settle commit: `insertEntry`, `insertUsage`,
`setValue(branchTip)`, plus `deleteList(pendingAssistantFrames(...))`.
- `terminal.ts:50` `operationCleanupWrites` — bundles main-log deletes with
`toolMemos` and `toolOutputs` deletes, returned into the same array as
`operationResultValue` and `laneStateValue`.
- `tools.ts:230, 257` — `deleteValue(pendingToolOutput)` with main-log writes.
- `deferred.ts:157` — `deleteList(pendingAssistantFrames)`.
**The adjustment: retirement becomes a main-log record.**
```ts
retireScope(operationId): Write<SessionScope>
```
The log is the truth; the sidecar file is a cache. Recovery reads the retire record
and ignores — then removes — the sidecar. Unlinking eagerly after commit is a pure
optimisation; losing the unlink costs disk, never correctness.
This makes all four sites above single-scope, because the individual ephemeral
deletes disappear. It also *simplifies* `operationCleanupWrites`: the `scanValues`
calls for `operationToolMemoPrefix` and `pendingToolOutputPrefix` are replaced by
one `retireScope`.
And it makes the open-time sweep principled rather than heuristic. A sidecar is
dead iff the main log retired it or its operation is absent — no inference from
operation status.
### 5.1 Two ephemeral scopes in one transaction cannot arise
A lane holds `operation: { meta, state } | null` — **singular**. Every write set
uses one `operationId`, either `drive.operationId` or the newly created one in
`startOperation`, which asserts no operation is active. Retire and start are always
separate transactions: the settle commits at `lane.ts:427` and `response.ts:339`
set lane state to `null`.
Concurrency is across **lanes**, and each lane has at most one active operation, so
two ephemeral scopes never meet in a commit. The runtime assertion in §6 is defence
against a future change, not a live gap.
## 6. Static enforcement
> **`scopes.variance.ts` ships alongside this doc** and is the executable form of
> this section: `npx tsc --noEmit --strict --lib es2023 scopes.variance.ts`.
> Silence means the rule holds. Its three `@ts-expect-error` lines fail the
> compile if they ever start type-checking, so it catches the enforcement being
> weakened as well as broken.
Two phantom types, differing only in where `Sc` appears. This distinction is
load-bearing and is the single easiest thing to get wrong here.
```ts
declare const storedScopeType: unique symbol;
/** Addresses: COVARIANT — Sc only in return position. */
export interface ScopedAddress<Sc extends Scope> {
readonly [storedScopeType]?: () => Sc;
}
/** Writes: INVARIANT — Sc in both parameter and return position. */
export interface Scoped<Sc extends Scope> {
readonly [storedScopeType]?: (scope: Sc) => Sc;
}
export interface Value<T, Sc extends Scope = SessionScope>
extends StoredAddressBase, ScopedAddress<Sc> { … }
export interface ValueSetWrite<Sc extends Scope> extends Scoped<Sc> { … }
```
**Why they differ.** Reading an address is safe at any scope — `getValue` does not
care which file a value lives in, so `Value<T, SessionScope>` must be usable where
`Value<T, Scope>` is expected. Forming a transaction is *not* safe at any scope,
because two files are not atomic, so a commit must pin down exactly one.
Enforcement therefore belongs where transactions are formed, not where addresses
are read. `setValue<T, Sc>(address: Value<T, Sc>, …): Write<Sc>` is the bridge: it
infers the scope from a covariant address and stamps it onto an invariant write.
> **Getting this wrong is expensive.** Making addresses invariant too breaks every
> reader — `getValue`, `scanValues`, `readList` and everything downstream — and
> produced a 36-error cascade across storage backends that had done nothing wrong.
> The tempting workaround, making each reader generic over `Sc` to mean "any
> scope", propagates a type parameter through the whole storage stack to express
> something variance gives for free.
Verified with `tsc --noEmit --strict`. Reads pass at both scopes:
```ts
getValue(laneState); // Value<T, SessionScope>
getValue(pendingOutput); // Value<T, EphemeralScope>
```
Single-scope commits pass:
```ts
commit([setValue(laneState, a), setValue(operationState, b)]);
commit([setValue(toolMemo, m), setValue(pendingOutput, o)]);
commit([setValue(laneState, a), retireScope("op_1")]);
```
Mixed-scope commits fail, and are the *only* things that fail:
```ts
commit([setValue(laneState, a), setValue(pendingOutput, o)]); // ERROR
commit([setValue(pendingOutput, o), retireScope("op_1")]); // ERROR
```
### 6.1 What the type system cannot catch
Two different ephemeral scopes both type as `Write<EphemeralScope>`, because the id
is a runtime string (§2). A runtime assertion at commit covers it — compare
`scopeId` across all writes — and per §5.1 it should never fire.
## 7. Recovery and sweeping
On open, enumerate sidecars alongside the main log and load them. Their records
participate in replay exactly as main-log records do.
A sidecar is removed when the main log holds a `retireScope` record for it, or when
its operation is absent — a crash between commit and unlink. Torn-tail semantics
are unchanged: each line is self-contained, so a crash mid-write loses exactly the
trailing line.
## 8. Sequence numbers
Scoped writes get their own sequence space per scope. Nothing outside a scope
orders against its contents, and keeping them separate means `nextSeq` in the main
header does not account for numbers consumed by files that will be deleted.
Shared numbering would leave large gaps after several operations settled, and the
high-water mark would need separate persistence.
## 9. Measured effect
Same workload as §1:
| | size | of today |
| --- | --- | --- |
| today, single JSONL | 93.89 MB | — |
| ops + interned addresses, single JSONL | 5.32 MB | 5.7% |
| **plus scopes — main log** | **0.06 MB** | **0.06%** |
| — sidecars, retired on settle | 5.26 MB | peak 0.26 MB per operation |
The 5.7% figure is **rate-dependent and must not be quoted alone**: it holds at
2 KB of tool output per checkpoint and inverts above the cap, where a whole-value
replacement wins.
The scope result is **not** rate-dependent. The main log holds settled entries
regardless of encoding, and that is the number that governs session file growth.
## 10. What changes in the existing code
The compiler finds these for you: once addresses carry a scope and `Write<Sc>` is
invariant, every site that mixes scopes in one transaction fails to typecheck.
This list is what a full pass turned up, so you know when you are done.
**New, in `session/values.ts` and `session/types.ts`**
`SessionScope` / `EphemeralScope` / `Scope`; the two phantoms from §6;
`Value<T, Sc>` and `ValueList<T, Sc>`; `scopeId` on the address; the `value` /
`list` overloads; `retireScope`; scope-preserving `setValue` / `deleteValue` /
`appendList` / `deleteList`; `Write<Sc>` with entries, usage and retirement
session-only by construction.
**Generified**
`CommitDecision<TResult, Sc>`, `lane.command<TResult, Sc>`, `Storage.commit<Sc>`.
Give `Sc` a default of `SessionScope`: conditional types are not inference sites,
so without a default a mixed array falls back to the constraint `Scope` and every
call site fails. `settleOperation` and `continueOperation` stay **non**-generic —
see §5.
**Committed writes**
`CommittedScopeRetireWrite`, and `scopeId` carried through the committed shapes so
`JsonlStorage` can route on it.
**Call sites that must change** — each currently mixes scopes in one transaction:
| site | what it does today | what it becomes |
| --- | --- | --- |
| `drive/terminal.ts` `operationCleanupWrites` | enumerates tool memos and tool outputs via `scanValues`, deletes each alongside session deletes | one `retireScope(operationId)`; two of its four scans disappear |
| `drive/response.ts` (settle) | `deleteList(pendingAssistantOutput)` bundled with operation state | drop it — the sidecar is discarded at retire |
| `drive/deferred.ts` (superseded response) | same | same |
| `drive/tools.ts` (tool settle) | `deleteValue(pendingToolOutput)` plus memo deletes in a session write array | drop them |
| `drive/tool-placement.ts` | `deleteValue(pendingToolOutput)` | drop it |
The pattern is the same everywhere: **intra-operation cleanup of ephemeral state
is impossible**, because those commits also write operation and lane state. The
sidecar is discarded wholesale at retire. The cost is that a multi-turn operation
holds superseded pending output until settle — bounded by turns, and never
reaching the main log.
**Storage**
Sidecar routing in `JsonlStorage.commit` (assert one scope id per transaction,
route the append), unlink on a `scope` record **after** the commit, and an
open-time sweep that loads sidecars and removes any the main log already retired.
In-memory drops a `Map` keyed by scope id; SQLite issues one
`DELETE ... WHERE scope = ?` and treats a retire record as a no-op, because it has
no sidecar and the terminal transaction deletes the scoped values explicitly.
**Expected test churn**
Nine tests assert the old cleanup write set and will fail — `drive-terminal` (3),
`drive-retry-deferred` (2), `drive-tools`, `drive-reconcile`, `drive-generation`,
and one pinning reserved namespaces. They expect six or seven individual deletes
where the new code emits fewer plus one `retireScope`. That is the change landing,
not a regression.
## 11. List tags and stop conditions
Tracked state is stored as a list of op batches (`delta.md` §9). Recovery needs "the
batches since the last base batch" without unpacking every row, which lists cannot
express today: `ListReadOptions` has `cursor`, `order` and `limit` only.
`BranchScan` already solves the same problem for entries, and its vocabulary is
the one to copy — `stopAtType` / `stopAtId` for a terminator, plain field names
for a filter.
```ts
appendList<T, Sc>(address: ValueList<T, Sc>, element: T, tag?: string): ListAppendWrite<Sc>;
export interface ListElement<T> {
seq: number;
value: T;
tag?: string;
}
export interface ListReadOptions {
cursor?: ListCursor;
order?: "asc" | "desc";
limit?: number;
/** Include elements up to and including the first carrying this tag, then stop. */
stopAtTag?: string;
/** Return only elements carrying this tag. */
tag?: string;
}
```
**`stopAtTag` is a stop condition within a page, not a guarantee.** It can only end
a page earlier than `limit` would; it never overrides it. If the tagged element is
not in the page, the consumer sees no tagged element and pages again with the
cursor — the same loop `readAssistantFrames` already runs. There is no ordering
question between the two bounds and no "not found" error.
For that loop to work, `readList` must return the tag on each element. Otherwise a
consumer cannot tell "page ended at the tag" from "page ended at the limit"
without parsing the value, which is the thing the tag exists to avoid.
**The producer sets the tag, not storage.** For a tracked value the caller
already knows, because `isBase(ops)` is a token comparison on the first op
(`delta.md` §2):
```ts
const ops = tracker.flush();
if (ops.length === 0) return;
writes: [appendList(address, enc.encode(ops), isBase(ops) ? "base" : undefined)];
```
Storage must never inspect an element to derive a tag. That is what keeps op
batches opaque to the storage layer, and it is the same property that makes
`EntryScan.type` work: the discriminant is a stored column, not something derived
from the payload.
**The tag lives on the storage record**, beside `seq` and `value`:
```jsonc
["l",7,9,[["r",{…}]],"base"] // per §12.1
```
Storage must never parse an element to evaluate a predicate. That is what keeps
ops opaque to storage and the durable path free of domain knowledge
(`harness-tools.md` §7.7). It is also why `EntryScan.type` works: the discriminant
is a stored column, not something derived from the payload.
`order` stays `"asc" | "desc"` rather than `BranchScan`'s
`"newestFirst" | "oldestFirst"`. Branch order is semantic — a walk from a tip
through a tree. List order is over `seq`. Borrowing the branch words would imply a
traversal that is not happening.
Backends: SQLite gets a `tag` column and a real predicate. JSONL holds the list in
memory already, so it scans backwards and checks a field — no worse than today.
In-memory likewise.
The primitive generalises past frames: any list wanting "the tail since the last
checkpoint" gets it.
## 12. JSONL record encoding
Two dictionaries at two layers. They are the same trick and are independent: the
**address** dictionary is over `namespace` + `key` on the record; the **path**
dictionary is over paths *inside* a value's ops (`delta.md` §4).
### 12.1 Records
```
["@", addrId, namespace, key] address definition
["v", addrId, seq, wireOps] value write — WireOp[], delta.md §4
["l", addrId, seq, element, tag?] list append — element is WireOp[]
for a tracked value
["x", addrId, seq] delete (value or list)
["!", addrId, seq] retire an ephemeral scope
```
**Record verbs and op verbs are separate vocabularies read at different levels**,
but they must not look alike. `!` rather than `r` for retire: `r` is delta's
replace op, and a reader scanning a file should not have to remember which nesting
level they are at to know what a tuple means.
Retire takes an address id like everything else, defined by an `["@", id, …]`
whose namespace is the scope. That keeps one interning rule for the whole file
rather than a special case for one record.
Entries and usage rows keep their current keyed form: they are written once, never
repeat an address, and are read by machinery that has nothing to do with ops.
An address is defined on its **second** use, matching path interning — a
definition is pure overhead for an address written once, and a session has many of
those.
### 12.2 Worked example
Four writes across two addresses, before and after.
```jsonc
// today — 1250 bytes
{"kind":"value","op":"set","seq":7,"namespace":"pi.op.state","key":"01a04cf6-…","value":{"at":"starting","control":{…},"settings":{…},"latestAssistantEntryId":null}}
{"kind":"value","op":"set","seq":8,"namespace":"pi.lane.state","key":"main","value":{"currentOperationId":"01a04cf6-…",…}}
{"kind":"value","op":"set","seq":9,"namespace":"pi.op.state","key":"01a04cf6-…","value":{"at":"checkpoint",…}}
{"kind":"value","op":"set","seq":12,"namespace":"pi.op.state","key":"01a04cf6-…","value":{"at":"assistant.ready",…}}
```
```jsonc
// with both dictionaries — 547 bytes
["@",0,"pi.op.state","01a04cf6-…"]
["v",0,7,[["r",{"at":"starting","control":{…},"settings":{…},"latestAssistantEntryId":null}]]]
["@",1,"pi.lane.state","main"]
["v",1,8,[["r",{"currentOperationId":"01a04cf6-…",…}]]]
["v",0,9,[["#",0,["at"]],["s",0,"checkpoint"]]]
["v",0,12,[["s","assistant.ready"]]]
```
| line | before | after |
| --- | --- | --- |
| address definition | – | 61 |
| op.state -> `starting` (base batch) | 353 | 256 |
| address definition | – | 31 |
| lane.state (base batch) | 181 | 114 |
| op.state -> `checkpoint` | 355 | **48** |
| op.state -> `assistant.ready` | 361 | **37** |
| **total** | **1250** | **547** (44%) |
The shape matters more than the total. Base batches barely shrink — 353 to 256 is
envelope only, since the value ships whole either way. The transitions collapse
~8x because they carry one changed field instead of the whole state including the
never-changing `settings` block.
Both dictionaries appear on line 5: `0` in `["v",0,9,…]` is an **address** id,
while `["#",0,["at"]]` defines a **path** id inside the value's ops. Same trick,
different layers, separate tables. Line 6 then drops the path entirely — arity
omission, because it targets the same path as the previous op in that batch.
**Do not quote 44% as the file-level saving.** Dictionary entries are one-time, so
a real operation's ~11 op.state writes amortise them and the ratio improves; but
transcript entries are untouched, and they were 18 KB of a 67 KB tool-heavy run.
Applied there this hits the ~27 KB of value writes and would take the file down by
roughly a fifth, not by half.
### 12.3 Reading
A reader builds both tables while replaying, so a torn tail costs exactly the
trailing lines and no header needs rewriting. A snapshot rewrite starts fresh
tables and re-emits definitions naturally.
A numeric `addrId` for which no definition has been seen is a corrupt file, not a
recoverable state — unlike a missing path id, which cannot occur because path
definitions are inside the same record as their first use.
## 13. Open questions
- File-handle pressure across concurrent lanes, each with a live sidecar. Probably
fine, unmeasured.
- fsync policy for sidecars (§4) — matched to the main log, or relaxed given the
contents are bounded-loss scaffolding.
- Whether a long-running operation should rotate its sidecar. `rebase()`
(`delta.md` §3.2.3) bounds *recovery* length but not file size, so a command
running for hours still grows its sidecar without bound.
- Whether an in-place compaction pass for the main log is worth building anyway.
`createFromSnapshot` plus atomic replace is the mechanism; scopes reduce the need
but do not remove it, since superseded session-scoped values still accumulate.

65
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/02-scopes/scopes.variance.ts vendored
View File

@@ -1,65 +0,0 @@
/**
* Executable form of scopes.md §6.
*
* npx tsc --noEmit --strict --lib es2023 scopes.variance.ts
*
* Silence means the rule holds: reads accept either scope, single-scope commits
* pass, and mixed-scope commits are the ONLY thing rejected. The three
* `@ts-expect-error` lines fail the compile if they ever start type-checking,
* so this catches the enforcement being weakened as well as broken.
*
* The variance split is the part that is easy to get backwards. Making addresses
* invariant too breaks every reader — getValue, scanValues, readList and
* everything downstream — and the tempting fix, making each reader generic over
* Sc, propagates a type parameter through the whole storage stack to express
* what variance gives for free.
*/
declare const tag: unique symbol;
type SessionScope = { readonly kind: "session" };
type EphemeralScope = { readonly kind: "ephemeral" };
type Scope = SessionScope | EphemeralScope;
/** Addresses: COVARIANT — Sc only in return position. Readers take any scope. */
interface ScopedAddress<Sc extends Scope> { readonly [tag]?: () => Sc }
/** Writes: INVARIANT — Sc in both positions. Mixed commits cannot unify. */
interface Scoped<Sc extends Scope> { readonly [tag]?: (s: Sc) => Sc }
interface Value<T, Sc extends Scope = SessionScope> extends ScopedAddress<Sc> { k: "v" }
interface ValueList<T, Sc extends Scope = SessionScope> extends ScopedAddress<Sc> { k: "l" }
interface VSet<Sc extends Scope = Scope> extends Scoped<Sc> { kind: "value"; op: "set" }
interface VDel<Sc extends Scope = Scope> extends Scoped<Sc> { kind: "value"; op: "delete" }
interface LApp<Sc extends Scope = Scope> extends Scoped<Sc> { kind: "list"; op: "append" }
interface EntryWrite { kind: "entry" }
interface RetireWrite extends Scoped<SessionScope> { kind: "scope" }
type Write<Sc extends Scope = SessionScope> =
| VSet<Sc> | VDel<Sc> | LApp<Sc>
| (Sc extends SessionScope ? EntryWrite | RetireWrite : never);
declare function setValue<T, Sc extends Scope>(a: Value<T, Sc>, v: T): VSet<Sc>;
declare function deleteValue<T, Sc extends Scope>(a: Value<T, Sc>): VDel<Sc>;
declare function appendList<T, Sc extends Scope>(a: ValueList<T, Sc>, v: T): LApp<Sc>;
declare function retireScope(id: string): RetireWrite;
declare function insertEntry(e: unknown): EntryWrite;
declare function commit<Sc extends Scope = SessionScope>(w: readonly Write<Sc>[]): void;
declare function getValue<T>(a: Value<T, Scope>): T; // reader: any scope
const laneState = null as unknown as Value<string>; // session
const pendingOut = null as unknown as ValueList<string, EphemeralScope>; // ephemeral
const toolMemo = null as unknown as Value<string, EphemeralScope>;
// READS accept either scope — that is what covariance buys.
getValue(laneState);
getValue(toolMemo);
// Single-scope commits pass.
commit([setValue(laneState, "a")]);
commit([setValue(laneState, "a"), insertEntry({}), retireScope("op_1")]);
commit([appendList(pendingOut, "x"), setValue(toolMemo, "m")]);
// @ts-expect-error mixed scopes in one transaction
commit([setValue(laneState, "a"), appendList(pendingOut, "x")]);
// @ts-expect-error a retire is session-only, so it cannot join an ephemeral commit
commit([appendList(pendingOut, "x"), retireScope("op_1")]);
// @ts-expect-error entries are session-only
commit<EphemeralScope>([insertEntry({})]);

172
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/03-execenv/bash.after.ts vendored
View File

@@ -1,172 +0,0 @@
import { type Static, Type } from "typebox";
import type { Context } from "../context.ts";
import type { AgentHarnessTool } from "../types.ts";
import { getOrThrow } from "../types.ts";
import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult } from "../utils/truncate.ts";
import type { ExecutionToolContext } from "./tool-context.ts";
const MAX_TIMEOUT_SECONDS = 2_147_483_647 / 1000;
/**
* Coalescing cadence for live updates. The env owns capping and coalescing now
* (`executionenv.md` §3), so bash no longer has an update throttle or a
* checkpoint interval of its own.
*/
const BASH_UPDATE_INTERVAL_MS = 100;
const bashSchema = Type.Object({
command: Type.String({ description: "Bash command to execute" }),
timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })),
});
export type BashToolInput = Static<typeof bashSchema>;
export interface BashToolDetails {
truncation?: TruncationResult;
/** Set by the exec env, on the machine the command ran on. */
spillPath?: string;
}
export interface BashExecution {
command: string;
cwd: string;
env: Record<string, string>;
inheritEnv: boolean;
}
export type BashPrepare<TContext extends ExecutionToolContext = ExecutionToolContext> = (
execution: BashExecution,
toolContext: TContext,
context: Context,
) => void | Promise<void>;
export interface BashToolOptions<TContext extends ExecutionToolContext = ExecutionToolContext> {
commandPrefix?: string;
prepare?: BashPrepare<TContext>;
}
function validateTimeout(timeout: number | undefined): void {
if (timeout === undefined) return;
if (!Number.isFinite(timeout) || timeout <= 0) {
throw new Error("Invalid timeout: must be a finite number of seconds");
}
if (timeout > MAX_TIMEOUT_SECONDS) {
throw new Error(`Invalid timeout: maximum is ${MAX_TIMEOUT_SECONDS} seconds`);
}
}
export function createBashTool<TContext extends ExecutionToolContext = ExecutionToolContext>(
options?: BashToolOptions<TContext>,
): AgentHarnessTool<TContext, typeof bashSchema, BashToolDetails | undefined> {
return {
name: "bash",
label: "bash",
description: `Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`,
parameters: bashSchema,
async execute(_toolCallId, { command, timeout }, onUpdate, toolContext, _invocation, context) {
validateTimeout(timeout);
const { env } = toolContext;
const execution: BashExecution = {
command: options?.commandPrefix ? `${options.commandPrefix}\n${command}` : command,
cwd: env.cwd,
env: {},
inheritEnv: true,
};
await options?.prepare?.(execution, toolContext, context);
// The env caps, coalesces and spills at the source. bash keeps a mirror
// only to build the settled result; it never truncates and never chooses
// a spill path.
let view = "";
let truncation: TruncationResult | undefined;
let spillPath: string | undefined;
const publish = (checkpoint: boolean): void => {
onUpdate(
{
content: [{ type: "text" as const, text: view }],
details: { truncation, spillPath },
},
checkpoint ? { checkpoint: true } : undefined,
);
};
onUpdate({ content: [], details: undefined });
const result = await env.exec(
execution.command,
{
cwd: execution.cwd,
env: execution.env,
inheritEnv: execution.inheritEnv,
timeout,
capture: {
limits: { maxBytes: DEFAULT_MAX_BYTES, maxLines: DEFAULT_MAX_LINES, retain: "tail" },
intervalMs: BASH_UPDATE_INTERVAL_MS,
spill: true,
},
onUpdate: (update) => {
switch (update.kind) {
case "chunk":
view += update.text;
break;
case "snapshot":
view = update.output.text;
truncation = update.output.truncation;
spillPath = update.output.spillPath;
break;
case "counters":
truncation = update.truncation;
break;
}
// A snapshot is the durable-worthy state: it is the only kind that
// can express eviction, so it is where a crash loses least.
publish(update.kind === "snapshot");
},
},
context,
);
if (!result.ok) {
const error = result.error;
const status =
error.code === "timeout"
? `Command timed out after ${timeout} seconds`
: error.code === "aborted"
? "Command aborted"
: error.message;
throw new Error(view ? `${view}\n\n${status}` : status, { cause: error });
}
const captured = result.value.output;
view = captured.text;
truncation = captured.truncation;
spillPath = captured.spillPath;
let outputText = captured.text;
let details: BashToolDetails | undefined;
if (captured.truncation.truncated) {
details = { truncation: captured.truncation, spillPath };
const startLine = captured.truncation.totalLines - captured.truncation.outputLines + 1;
const endLine = captured.truncation.totalLines;
if (captured.truncation.lastLinePartial) {
// One line alone exceeded the byte cap. Line numbers would be
// misleading here, so report the line's own size instead — and only
// the env can measure it, since it holds the raw buffer.
const lastLineSize = formatSize(captured.lastLineBytes ?? captured.truncation.outputBytes);
outputText += `\n\n[Showing last ${formatSize(captured.truncation.outputBytes)} of line ${endLine} (line is ${lastLineSize}). Full output: ${spillPath}]`;
} else if (captured.truncation.truncatedBy === "lines") {
outputText += `\n\n[Showing lines ${startLine}-${endLine} of ${captured.truncation.totalLines}. Full output: ${spillPath}]`;
} else {
outputText += `\n\n[Showing lines ${startLine}-${endLine} of ${captured.truncation.totalLines} (${formatSize(DEFAULT_MAX_BYTES)} limit). Full output: ${spillPath}]`;
}
}
if (result.value.exitCode !== 0) {
throw new Error(
`${outputText ? `${outputText}\n\n` : ""}Command exited with code ${result.value.exitCode}`,
);
}
return { content: [{ type: "text", text: outputText || "(no output)" }], details };
},
};
}

148
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/03-execenv/execenv.md vendored
View File

@@ -1,148 +0,0 @@
# ExecutionEnv: bounded shell output
**Status:** implemented in production source. The prototype files beside this document are historical evidence; production lives in:
- `packages/agent/src/harness/utils/adaptive-publisher.ts`
- `packages/agent/src/harness/utils/output-capture.ts`
- `packages/agent/src/harness/env/nodejs.ts`
- `packages/agent/src/harness/tools/bash.ts`
The generic `ToolOutput` use of the same publisher remains in `04-tool-output`.
## 1. Problem
The old `Shell.exec()` accumulated complete strings inside `NodeExecutionEnv`:
```ts
stdout += chunk;
stderr += chunk;
```
Bash truncated only after those strings had already been built. `cat 1gb.txt` therefore materialized a gigabyte in the worker before any tool-level bound could help.
Spilling also belongs where bytes originate. If execution is remote, a spill on the worker is inaccessible to the model's `read` and `grep` tools, and creating it after transport would first send the complete gigabyte over the connection.
## 2. Boundary
`ExecutionEnv` now owns:
- a bounded head or tail view;
- complete byte and line totals;
- lazy source-local spill after the view first crosses its limits;
- persistent source-local spill with bounded write-stream backpressure;
- adaptive publication of the latest bounded state;
- a forced final publication before settlement.
It does not return or retain separate `stdout` and `stderr` values. Both pipes feed one arrival-ordered model-visible text view, matching bash and `ToolOutput`; preserving stream styling through tail eviction would require a segmented retained state that the Harness does not expose. Text is folded from updates, and `ShellExecResult` contains only exit and truncation/spill metadata.
Bash owns only command semantics and its model-visible footer. Its old rolling buffer, spill creation, 100 ms throttle, and full-output accumulation are gone. The existing two-second durable checkpoint request remains temporarily until `ToolOutput` owns durable cadence.
## 3. Contract
```ts
interface ShellOutputLimits {
maxBytes: number;
maxLines: number;
retain?: "head" | "tail";
}
interface ShellOutputCaptureOptions {
limits: ShellOutputLimits;
spill?: boolean;
}
type ShellOutputTruncation = Omit<TruncationResult, "content">;
interface ShellOutputMetadata {
truncation: ShellOutputTruncation;
spillPath?: string;
lastLineBytes?: number;
}
interface ShellOutputView extends ShellOutputMetadata {
text: string;
}
type ShellOutputUpdate =
| { kind: "replace"; output: ShellOutputView }
| { kind: "append"; text: string; metadata: ShellOutputMetadata }
| { kind: "slide"; drop: number; text: string; metadata: ShellOutputMetadata }
| { kind: "metadata"; metadata: ShellOutputMetadata };
interface ShellExecResult extends ShellOutputMetadata {
exitCode: number;
}
```
`drop` counts JavaScript string code units, matching `slice()`. Updates are ordered. A consumer applies them with `applyShellOutputUpdate()`.
A complete replacement establishes the initial state or recovers when no verified overlap exists. An append carries only a growing suffix. A slide drops a prefix and appends the new suffix. Metadata updates move totals or the spill path without resending text.
The compatibility `executeShellWithCapture()` helper still returns one bounded final view. Its `onChunk` callback receives initial, append, and slide text only; metadata and post-turnover replacements are not mislabeled as new bytes.
## 4. Adaptive publication
`AdaptivePublisher` keeps only the latest dirty state. Intermediate process writes never become a queue of output updates.
Policy is harness-global rather than per tool:
```ts
minIntervalMs = 100;
targetBytesPerSecond = 100 * 1024;
nextDelayMs = max(minIntervalMs, encodedUpdateBytes * 1000 / targetBytesPerSecond);
```
The first dirty state after idle is immediate. Writes received before the deadline collapse into the latest bounded state. One trailing timer guarantees eventual publication. Finalization bypasses the deadline once, still bounded by the retained cap.
The publisher commits its baseline before invoking the consumer. If a consumer applies an update and then throws, finalization cannot emit the same delta twice. The command fails with `callback_error`.
### Workload behavior
| workload | result |
| --- | --- |
| Finishes below cap | immediate initial state, small appends, forced final state |
| Below-cap trickle | isolated writes are immediate; sustained writes are at most 100 ms apart |
| Full-force output after cap | complete turnovers are cap-sized replacements spaced by their encoded size |
| Post-cap trickle | small verified `slide` updates remain responsive; the complete window is not resent |
| Burst then silence | one leading update and one trailing update |
| Error, timeout, or abort | latest bounded state is forced before the error settles |
At a 50 KB cap and 100 KB/s target, repeated complete turnovers settle near two updates per second. Small post-cap slides still use the 100 ms floor.
The rate bound is amortized. An immediate-after-idle update and forced terminal update may each create one cap-bounded burst.
## 5. Capture and spill
`OutputCapture` trims its decoded line-aware working buffer back to twice the byte cap when it exceeds four times the cap. Tail mode drops old text; head mode preserves the original prefix. This amortizes UTF-8 trimming instead of rescanning the retained window for every process chunk.
Spill creation is lazy. Before crossing, the source keeps at most the bounded prefix needed to create a complete archive. On crossing it:
1. pauses stdout and stderr while it creates the file inside the execution environment;
2. opens one persistent append stream with a bounded 8 MB high-water mark;
3. writes the preserved raw prefix and subsequent raw chunks in arrival order;
4. resumes immediately while the writer accepts data;
5. pauses again only when `write()` reports backpressure, then resumes on `drain`.
This avoids both an unbounded promise chain and one async file-open/append cycle per process chunk. A spill create or stream-write failure kills the child and fails execution rather than publishing lossy success.
The spill path is force-published as metadata when it becomes available. Spill writes are awaited before the final output flush.
Node streams remain raw for spill throughput and exact archival bytes. `OutputCapture` uses one streaming `TextDecoder`, so a read boundary cannot split a code point; invalid display control characters are removed only from bounded snapshots, not by scanning the complete raw stream. Line totals count a final unterminated line, and `lastLineBytes` remains exact even when one line exceeds the working buffer.
## 6. Remote execution
When worker and execution environment are colocated, updates are in-process and this publisher primarily bounds capture work. `ToolOutput` remains the downstream event/durability limiter.
When an execution environment is physically remote from its worker, the same bounded updates cross that transport. Full-force output cannot transfer the complete stream: intermediate writes collapse at the source, while the complete stream remains in the source-local spill.
Each real costly boundary gets its own publisher instance. A colocated deployment may bypass transport serialization; a custom tool that bypasses `ExecutionEnv` still passes through the future `ToolOutput` publisher.
## 7. Remaining work
- Move generic custom-tool composition, text retention, event publication, and durable cadence into `ToolOutput`.
- Replace whole `AgentToolResult` progress with Chord operations at that downstream boundary.
- Make memo and output checkpoint persistence one atomic transaction.
- Seed replay from durable output rather than deleting it.
- Put spills in an environment-owned session directory and sweep them after session lifetime plus a crash-retention floor.
- Decide explicit limits for images and structured details; text is bounded, those values are not yet.
- Define raw binary-output behavior. Current shell output remains lossy UTF-8 text.

137
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/03-execenv/nodejs-env.diff vendored
View File

@@ -1,137 +0,0 @@
diff --git a/packages/agent/src/harness/env/nodejs.ts b/packages/agent/src/harness/env/nodejs.ts
index 732dc84db..db9eb9a57 100644
--- a/packages/agent/src/harness/env/nodejs.ts
+++ b/packages/agent/src/harness/env/nodejs.ts
@@ -19,6 +19,7 @@ import { basename, isAbsolute, join, resolve } from "node:path";
import { createInterface } from "node:readline";
import { fileURLToPath } from "node:url";
import type { Context } from "../context.ts";
+import { OutputCapture } from "../utils/output-capture.ts";
import {
type ExecutionEnv,
ExecutionError,
@@ -28,7 +29,10 @@ import {
type FileKind,
ok,
type Result,
+ type ShellCapabilities,
type ShellExecOptions,
+ type ShellExecResult,
+ type ShellStream,
toError,
} from "../types.ts";
@@ -365,11 +369,15 @@ export class NodeExecutionEnv implements ExecutionEnv {
return ok(join(...parts));
}
+ capabilities(): ShellCapabilities {
+ return { capture: true, spill: true, liveUpdates: true };
+ }
+
async exec(
command: string,
options: ShellExecOptions | undefined,
context: Context,
- ): Promise<Result<{ stdout: string; stderr: string; exitCode: number }, ExecutionError>> {
+ ): Promise<Result<ShellExecResult, ExecutionError>> {
const signal = context.abortSignal;
if (signal?.aborted) return err(new ExecutionError("aborted", "aborted"));
const timeoutMsResult = resolveTimeoutMs(options?.timeout);
@@ -392,9 +400,28 @@ export class NodeExecutionEnv implements ExecutionEnv {
);
}
+ // Spill lives in the exec env, on the machine where the model's own `read`
+ // and `grep` run — the whole reason capping happens here rather than on the
+ // agent side (`executionenv.md` §1).
+ let spillHandle: string | undefined;
+ let spillChain: Promise<void> = Promise.resolve();
+ const wantSpill = options?.capture?.spill === true;
+
return await new Promise((resolvePromise) => {
- let stdout = "";
- let stderr = "";
+ const capture = new OutputCapture(options?.capture, options?.onUpdate);
+ const spillChunk = (chunk: string): Promise<void> => {
+ if (!wantSpill) return Promise.resolve();
+ spillChain = spillChain.then(async () => {
+ if (spillHandle === undefined) {
+ const created = await this.createTempFile({ prefix: "pi-output-", suffix: ".log" }, context);
+ if (!created.ok) return;
+ spillHandle = created.value;
+ capture.setSpillPath(spillHandle);
+ }
+ await this.appendFile(spillHandle, chunk, context);
+ });
+ return spillChain;
+ };
let settled = false;
let timedOut = false;
let callbackError: ExecutionError | undefined;
@@ -407,7 +434,7 @@ export class NodeExecutionEnv implements ExecutionEnv {
}
};
- const settle = (result: Result<{ stdout: string; stderr: string; exitCode: number }, ExecutionError>) => {
+ const settle = (result: Result<ShellExecResult, ExecutionError>) => {
if (timeoutId) clearTimeout(timeoutId);
if (signal) signal.removeEventListener("abort", onAbort);
if (child?.pid) this.activeChildPids.delete(child.pid);
@@ -458,31 +485,32 @@ export class NodeExecutionEnv implements ExecutionEnv {
}
}
+ // setEncoding("utf8") is what keeps a chunk boundary from splitting a
+ // codepoint; without it a multi-byte sequence straddling a read bakes in a
+ // replacement character (`executionenv.md` §8).
child.stdout?.setEncoding("utf8");
child.stderr?.setEncoding("utf8");
- child.stdout?.on("data", (chunk: string) => {
- stdout += chunk;
- try {
- options?.onStdout?.(chunk, context);
- } catch (error) {
- const cause = toError(error);
- callbackError = new ExecutionError("callback_error", cause.message, cause);
- onAbort();
- }
- });
- child.stderr?.on("data", (chunk: string) => {
- stderr += chunk;
+ const feed = (stream: ShellStream, chunk: string) => {
try {
- options?.onStderr?.(chunk, context);
+ capture.push(stream, chunk, context);
+ void spillChunk(chunk);
} catch (error) {
const cause = toError(error);
callbackError = new ExecutionError("callback_error", cause.message, cause);
onAbort();
}
- });
+ };
+ child.stdout?.on("data", (chunk: string) => feed("stdout", chunk));
+ child.stderr?.on("data", (chunk: string) => feed("stderr", chunk));
void waitForChildProcess(child).then(
- (code) => {
+ async (code) => {
+ // The spill chain is async and lazy: the file is created on the first
+ // chunk. Await it before snapshotting, or the returned CapturedOutput
+ // reports no spillPath for output that was in fact archived.
+ await spillChain;
+ capture.flush(context);
+ capture.dispose();
if (callbackError) {
settle(err(callbackError));
return;
@@ -495,7 +523,7 @@ export class NodeExecutionEnv implements ExecutionEnv {
settle(err(new ExecutionError("aborted", "aborted")));
return;
}
- settle(ok({ stdout, stderr, exitCode: code ?? 0 }));
+ settle(ok({ output: capture.snapshot(), exitCode: code ?? 0 }));
},
(error: Error) => settle(err(new ExecutionError("spawn_error", error.message, error))),
);

149
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/03-execenv/nodejs-env.test.diff vendored
View File

@@ -1,149 +0,0 @@
diff --git a/packages/agent/test/harness/nodejs-env.test.ts b/packages/agent/test/harness/nodejs-env.test.ts
index 361040495..ee28ef506 100644
--- a/packages/agent/test/harness/nodejs-env.test.ts
+++ b/packages/agent/test/harness/nodejs-env.test.ts
@@ -8,7 +8,6 @@ import { afterEach, describe, expect, it } from "vitest";
import { BACKGROUND_CONTEXT, withAbortSignal } from "../../src/harness/context.ts";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { FileError, getOrThrow } from "../../src/harness/types.ts";
-import { executeShellWithCapture } from "../../src/harness/utils/shell-output.ts";
import { createTempDir } from "./session-test-utils.ts";
const chmodRestorePaths: string[] = [];
@@ -303,7 +302,8 @@ describe("NodeExecutionEnv", () => {
BACKGROUND_CONTEXT,
),
);
- expect(result).toEqual({ stdout: `${await realpath(root)}:ok`, stderr: "", exitCode: 0 });
+ expect(result.output.text).toContain(`${await realpath(root)}:ok`.trim());
+ expect(result.exitCode).toBe(0);
});
it.each([
@@ -334,7 +334,7 @@ describe("NodeExecutionEnv", () => {
),
);
- expect(result.stdout).toBe(`${expectedSessionFile}|true|preserved`);
+ expect(result.output.text).toBe(`${expectedSessionFile}|true|preserved`);
},
);
@@ -358,7 +358,7 @@ describe("NodeExecutionEnv", () => {
),
);
- expect(result.stdout).toBe("::explicit");
+ expect(result.output.text).toBe("::explicit");
} finally {
if (previousInherited === undefined) delete process.env[inheritedKey];
else process.env[inheritedKey] = previousInherited;
@@ -396,7 +396,9 @@ describe("NodeExecutionEnv", () => {
await wslEnv.exec(`name='World'; echo "Hello, ${nameExpansion}!"`, undefined, BACKGROUND_CONTEXT),
);
- expect(result).toEqual({ stdout: "Hello, World!\n", stderr: "args:-s\n", exitCode: 0 });
+ expect(result.output.text).toContain("Hello, World!");
+ expect(result.output.text).toContain("args:-s");
+ expect(result.exitCode).toBe(0);
} finally {
process.chdir(originalCwd);
process.env.PATH = originalPath;
@@ -425,7 +427,7 @@ describe("NodeExecutionEnv", () => {
() => controller.abort(),
),
);
- expect(result.stdout).toContain("child-exiting");
+ expect(result.output.text).toContain("child-exiting");
} finally {
controller.abort();
cleanupDetachedChild(pidFile);
@@ -448,25 +450,30 @@ describe("NodeExecutionEnv", () => {
it("streams stdout and stderr chunks", async () => {
const root = createTempDir();
const env = new NodeExecutionEnv({ cwd: root });
- let stdout = "";
- let stderr = "";
+ // Streams are aggregated in arrival order now; `chunk` carries `stream` so a
+ // renderer can still colour stderr (`executionenv.md` §6).
+ let seen = "";
+ const streams: string[] = [];
const result = getOrThrow(
await env.exec(
"printf out; printf err >&2",
{
- onStdout: (chunk) => {
- stdout += chunk;
- },
- onStderr: (chunk) => {
- stderr += chunk;
+ onUpdate: (update) => {
+ if (update.kind !== "chunk") return;
+ seen += update.text;
+ streams.push(update.stream);
},
},
BACKGROUND_CONTEXT,
),
);
- expect(result).toEqual({ stdout: "out", stderr: "err", exitCode: 0 });
- expect(stdout).toBe("out");
- expect(stderr).toBe("err");
+ expect(result.exitCode).toBe(0);
+ expect(result.output.text).toContain("out");
+ expect(result.output.text).toContain("err");
+ expect(seen).toContain("out");
+ expect(seen).toContain("err");
+ expect(streams).toContain("stdout");
+ expect(streams).toContain("stderr");
});
it("reports a missing working directory before spawning", async () => {
@@ -484,7 +491,8 @@ describe("NodeExecutionEnv", () => {
const root = createTempDir();
const env = new NodeExecutionEnv({ cwd: root });
const result = getOrThrow(await env.exec("exit 7", undefined, BACKGROUND_CONTEXT));
- expect(result).toEqual({ stdout: "", stderr: "", exitCode: 7 });
+ expect(result.output.text).toContain("".trim());
+ expect(result.exitCode).toBe(7);
});
it("returns timeout errors for commands exceeding the timeout", async () => {
@@ -501,7 +509,7 @@ describe("NodeExecutionEnv", () => {
const result = await env.exec(
"printf out",
{
- onStdout: () => {
+ onUpdate: () => {
throw new Error("callback failed");
},
},
@@ -542,12 +550,23 @@ describe("NodeExecutionEnv", () => {
const root = createTempDir();
const env = new NodeExecutionEnv({ cwd: root });
const result = getOrThrow(
- await executeShellWithCapture(env, "yes line | head -n 15000", undefined, BACKGROUND_CONTEXT),
+ await env.exec(
+ "yes line | head -n 15000",
+ {
+ capture: {
+ limits: { maxBytes: 50 * 1024, maxLines: 2000, retain: "tail" },
+ spill: true,
+ },
+ },
+ BACKGROUND_CONTEXT,
+ ),
);
- expect(result.truncated).toBe(true);
- expect(result.fullOutputPath).toBeDefined();
- const fullOutput = getOrThrow(await env.readTextFile(result.fullOutputPath!, BACKGROUND_CONTEXT));
+ expect(result.output.truncation.truncated).toBe(true);
+ // The spill lives inside the exec env, on the machine the command ran on —
+ // which is where the model's own read and grep run (`executionenv.md` §1).
+ expect(result.output.spillPath).toBeDefined();
+ const fullOutput = getOrThrow(await env.readTextFile(result.output.spillPath!, BACKGROUND_CONTEXT));
expect(fullOutput.split("\n").length).toBeGreaterThan(10000);
- expect(result.output.length).toBeLessThan(fullOutput.length);
+ expect(result.output.text.length).toBeLessThan(fullOutput.length);
});
});

194
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/03-execenv/output-capture.ts vendored
View File

@@ -1,194 +0,0 @@
import type { Context } from "../context.ts";
import type { CaptureOptions, CapturedOutput, ShellOutputUpdate, ShellStream } from "../types.ts";
import {
DEFAULT_MAX_BYTES,
DEFAULT_MAX_LINES,
type TruncationResult,
truncateHead,
truncateTail,
utf8ByteLength,
} from "./truncate.ts";
/**
* Bounded capture for a Shell implementation.
*
* Owns the two-stage structure that `shell-output.ts` had inside `bash.ts`: a raw
* buffer trimmed to `maxBytes * 2` as a memory guard, and a retained view
* computed from it per snapshot. `truncateTail` is line-aware — it walks
* backwards line by line and takes a partial only when one line alone exceeds the
* cap — so eviction is not byte arithmetic and cannot be done incrementally.
*
* Emits the three update kinds from `executionenv.md` §4 and enforces the
* ordering invariant: once the cap is crossed, a `chunk` never follows without an
* intervening `snapshot`.
*/
export class OutputCapture {
readonly #maxBytes: number;
readonly #maxLines: number;
readonly #retain: "head" | "tail";
readonly #intervalMs: number;
readonly #onUpdate?: (update: ShellOutputUpdate, context: Context) => void;
/** Raw rolling buffer, held at twice the cap. */
#buffer = "";
/** Everything ever produced. */
#totalBytes = 0;
/** Newlines seen. A trailing partial line is counted separately — a command
* emitting one 60 KB line with no newline has produced one line, not zero. */
#newlines = 0;
#endsWithNewline = true;
/** True once the retained view has begun evicting (tail) or filled (head). */
#crossed = false;
/** Pending chunks for the current coalescing window. */
#pending: { stream: ShellStream; text: string }[] = [];
#lastFlush = 0;
#timer: ReturnType<typeof setTimeout> | undefined;
#spillPath: string | undefined;
constructor(
options: CaptureOptions | undefined,
onUpdate: ((update: ShellOutputUpdate, context: Context) => void) | undefined,
) {
this.#maxBytes = options?.limits.maxBytes ?? DEFAULT_MAX_BYTES;
this.#maxLines = options?.limits.maxLines ?? DEFAULT_MAX_LINES;
this.#retain = options?.limits.retain ?? "tail";
this.#intervalMs = options?.intervalMs ?? 0;
this.#onUpdate = onUpdate;
}
push(stream: ShellStream, text: string, context: Context): void {
if (text === "") return;
this.#totalBytes += utf8ByteLength(text);
this.#newlines += countNewlines(text);
this.#endsWithNewline = text.endsWith("\n");
this.#buffer += text;
// Memory guard. The retained view is computed from this, never equal to it.
const guard = this.#maxBytes * 2;
if (utf8ByteLength(this.#buffer) > guard) {
this.#buffer = trimToLastBytes(this.#buffer, guard);
}
this.#pending.push({ stream, text });
this.#schedule(context);
}
setSpillPath(path: string | undefined): void {
this.#spillPath = path;
}
/** The bounded view, recomputed from the raw buffer. */
snapshot(): CapturedOutput {
const truncation =
this.#retain === "head"
? truncateHead(this.#buffer, { maxBytes: this.#maxBytes, maxLines: this.#maxLines })
: truncateTail(this.#buffer, { maxBytes: this.#maxBytes, maxLines: this.#maxLines });
// The buffer already lost its front, so totals come from counters rather
// than from what survives.
const totalLines = this.#totalLines();
const truncated = this.#totalBytes > this.#maxBytes || totalLines > this.#maxLines;
return {
text: truncation.content,
truncation: {
...truncation,
lastLinePartial: truncation.lastLinePartial,
truncated,
truncatedBy: truncated ? (totalLines > this.#maxLines ? "lines" : "bytes") : null,
totalBytes: this.#totalBytes,
totalLines,
},
...(this.#spillPath === undefined ? {} : { spillPath: this.#spillPath }),
...(truncation.lastLinePartial ? { lastLineBytes: this.#lastLineBytes() } : {}),
};
}
#totalLines(): number {
return this.#newlines + (this.#endsWithNewline || this.#totalBytes === 0 ? 0 : 1);
}
/** Bytes in the final line of the raw buffer, for the single-oversized-line case. */
#lastLineBytes(): number {
const start = this.#buffer.lastIndexOf("\n");
return utf8ByteLength(start === -1 ? this.#buffer : this.#buffer.slice(start + 1));
}
#schedule(context: Context): void {
if (this.#onUpdate === undefined) return;
if (this.#intervalMs === 0) {
this.flush(context);
return;
}
if (this.#timer !== undefined) return;
const wait = Math.max(0, this.#lastFlush + this.#intervalMs - Date.now());
this.#timer = setTimeout(() => {
this.#timer = undefined;
this.flush(context);
}, wait);
}
/** Emit one update for everything buffered since the last flush. */
flush(context: Context): void {
if (this.#onUpdate === undefined) return;
this.#lastFlush = Date.now();
const pending = this.#pending;
this.#pending = [];
if (pending.length === 0) return;
const nowCrossed = this.#totalBytes > this.#maxBytes || this.#totalLines() > this.#maxLines;
if (this.#retain === "head") {
// Head never evicts. Below the cap the appended text is complete; above
// it, only totals move — resending maxBytes to change a number is the
// thing `counters` exists to avoid.
if (!nowCrossed) {
this.#onUpdate({ kind: "chunk", stream: pending[0]!.stream, text: joined(pending) }, context);
} else {
this.#onUpdate({ kind: "counters", truncation: this.snapshot().truncation }, context);
}
this.#crossed = nowCrossed;
return;
}
// Tail. Once the buffer is full every byte in evicts a byte out, so there is
// no cheap-trickle regime past the cap: chunks until full, snapshots after.
if (nowCrossed) {
this.#crossed = true;
this.#onUpdate({ kind: "snapshot", output: this.snapshot() }, context);
return;
}
if (this.#crossed) {
// Defensive: never let a chunk follow the crossing.
this.#onUpdate({ kind: "snapshot", output: this.snapshot() }, context);
return;
}
this.#onUpdate({ kind: "chunk", stream: pending[0]!.stream, text: joined(pending) }, context);
}
dispose(): void {
if (this.#timer !== undefined) clearTimeout(this.#timer);
this.#timer = undefined;
}
}
const joined = (pending: { text: string }[]): string => pending.map((p) => p.text).join("");
function countNewlines(text: string): number {
let count = 0;
for (let index = text.indexOf("\n"); index !== -1; index = text.indexOf("\n", index + 1)) count++;
return count;
}
/** Keep the last `bytes` UTF-8 bytes without splitting a codepoint. */
function trimToLastBytes(text: string, bytes: number): string {
if (utf8ByteLength(text) <= bytes) return text;
let low = 0;
let high = text.length;
while (low < high) {
const mid = Math.floor((low + high) / 2);
if (utf8ByteLength(text.slice(mid)) > bytes) low = mid + 1;
else high = mid;
}
return text.slice(low);
}
export type { TruncationResult };

211
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/bash-worked-example.md vendored
View File

@@ -1,211 +0,0 @@
# Worked Example: `bash` End to End
One tool through every layer. `execute` is simplified — timeout validation,
cancellation and exit-code branches are elided. Everything on the output path is
shown.
Layers: exec env → `ToolOutput` sink → harness events → durable storage → lane
state → facet → wire → consumer.
---
## 1. The tool
```ts
// packages/agent/src/harness/tools/bash.ts
export interface BashToolDetails { spillPath?: string; truncation?: ShellOutputTruncation }
export function createBashTool(): AgentHarnessTool<ExecutionToolContext, typeof bashSchema, BashToolDetails> {
return {
name: "bash",
parameters: bashSchema,
output: { retain: "tail", maxBytes: DEFAULT_MAX_BYTES, maxLines: DEFAULT_MAX_LINES },
async execute(_id, { command, timeout }, signal, out, context) {
const env = context.env;
let view: ShellOutputView | undefined;
const result = getOrThrow(await env.exec(command, {
cwd: env.cwd,
inheritEnv: true,
timeout,
capture: { limits: this.output, spill: true },
onUpdate: (u) => {
view = applyShellOutputUpdate(view, u);
if (u.kind === "append") out.write(u.text);
else out.replace(view.text);
out.details.truncation = view.truncation;
if (view.spillPath) out.details.spillPath = view.spillPath;
},
}, context));
if (result.spillPath) out.details.spillPath = result.spillPath;
if (result.truncation.truncated) out.write(`\n\n[${describe(result.truncation)}]`);
if (result.exitCode) throw new Error(`Command exited with code ${result.exitCode}`);
},
};
}
```
What is gone from today's implementation: the rolling `tailOutput` buffer,
`truncateTail`, `ensureFullOutputFile`, `createTempFile`,
`BASH_UPDATE_THROTTLE_MS`, `BASH_CHECKPOINT_INTERVAL_MS`, `updateDirty`,
`lastCheckpoint`, `scheduleOutputUpdate`, `emitOutputUpdate`,
`clearUpdateTimer`. About 40 lines of tool-local machinery, replaced by a capture
policy every tool gets.
`execute` returns `void`. `details` is one field, set once — and note the tool no
longer computes `truncation`, because the env owns the window and the tool no
longer knows what was dropped.
## 2. The exec env caps at the source
`env.exec` applies `ShellOutputLimits` where the bytes originate. For a sandbox host
that means `cat 1gb.txt` never ships 1 GB to the agent machine, and the spill
lands where the model's own `read` and `grep` run. See [`execenv.md`](../03-execenv/execenv.md).
The initial state is a bounded `replace`. Growth emits `append`; a moving tail emits `slide { drop, text }`; complete turnover falls back to a bounded `replace`; metadata moves totals and spill paths without resending text. The adaptive publisher keeps small slides responsive and spaces cap-sized turnovers according to their encoded size.
## 3. The sink
`ToolOutput` folds those into `ToolOutputState`:
```ts
{
content: [{ type: "text", text: "…the retained window…" }],
details: { spillPath: "/tmp/pi-session-x/bash-8f2.log" },
usage: undefined,
addedTools: undefined,
terminate: false,
truncation: { truncated: true, truncatedBy: "bytes", totalLines: 8123, totalBytes: 262144 },
}
```
`out.write(text)` appends to `content[0].text`; `out.replace(text)` assigns
the whole retained view when the env sends a `snapshot`, because an append cannot
express eviction. The tracker (`delta.md`) records intent either way: an append is
an `a`, and a window slide assigned wholesale becomes `t` + `a` via verified
overlap detection.
## 4. Ops
A 256 KB build over ~20 flushes, 50 KB window:
The first batch is a base batch — it begins with `r`, carrying the initial state. Everything
after is deltas. The repeated content path is interned on its second use, and
consecutive ops on it omit the id entirely:
```jsonc
[["r",{"content":[{"type":"text","text":""}],"details":{},"terminate":false,"truncation":{…}}]]
[["a",["content",0,"text"],"make: Entering directory …\n"]]
[["#",0,["content",0,"text"]],["a",0,"cc -c src/a.c …\n"]]
[["a",0,"cc -c src/b.c …\n"]]
…
[["s",["details","spillPath"],"/tmp/pi-session-x/bash-8f2.log"]]
…
[["t",0,4096],["a","cc -c src/z.c\n"],["s",["truncation","totalBytes"],262144]]
```
Details are one `s`, in one flush out of twenty. The `t` + `a` pair is the window slide. The current tracker recovers it by verified overlap; production remeasurement shows the generic path is already negligible, so no explicit append/truncate producer API is added ([decision](../01-delta/append-decision.md)).
## 5. Harness events
```ts
{ type: "tool_start", toolCallId: "call_7", toolName: "bash",
args: { command: "make -j8" } }
{ type: "tool_update", toolCallId: "call_7", ops: [ … ] }
{ type: "tool_end", toolCallId: "call_7", isError: false }
```
`tool_start` is identity only — no `caps`, no `initial`. The first batch is a base
batch, so the initial state arrives on the update channel, and the caps are
already inside it as `truncation.maxBytes` / `maxLines`.
`tool_end` carries no content and no details. Every byte already went out;
re-sending would duplicate any images.
## 6. Durable storage
`pendingToolOutput(operationId, "call_7")`, **ephemeral-scoped** to the operation,
so it lives in `<session>.op_….jsonl` and is retired on settle rather than
persisting in the main log forever. Retirement is a main-log `retireScope` record,
so it commits atomically with the settle writes; the unlink is a consequence of
replaying that record, not part of the transaction ([scopes.md §5](../02-scopes/scopes.md)).
A `list<WireOp[]>`, one encoded batch appended per flush, base batches tagged `"base"` so recovery reads backwards with `stopAtTag` and stops there ([scopes.md §11](../02-scopes/scopes.md#11-list-tags-and-stop-conditions)). The tracker emits structural ops; the producer calls `rebase()` periodically to write a capped root replacement and bound recovery replay. The durable interval is a sink policy. Shell capture controls only source-state and transport publication; memo, terminal, and recovery-base flushes are forced by `ToolOutput`.
Recovery seeds a fresh `ToolOutput` from that state — it does **not** delete it,
which is what `clearReplayCheckpoint` does today and is a bug
(`harness-tools.md` §7.4). If bash memoised "spilled to /tmp/…" and crashed, a
replay that discarded the state would create a second spill file and lose the
first.
The memo write and this checkpoint commit in the same transaction. Both are
ephemeral-scoped, so both land in the same sidecar — and the type system rejects a
commit that mixes scopes, so this cannot regress silently.
## 7. Lane state and the facet
```ts
export function reduceLaneSnapshot(view: LaneView, event: HarnessEvent): void {
switch (event.type) {
case "tool_start":
view.operation.tools.push({ id: event.toolCallId, name: event.toolName,
args: event.args, output: undefined });
return;
case "tool_update": {
const tool = view.operation.tools.find((t) => t.id === event.toolCallId);
if (tool === undefined) return; // host will send a base batch
tool.output = apply(tool.output, event.ops);
return;
}
case "tool_end": {
const i = view.operation.tools.findIndex(t => t.id === event.toolCallId);
if (i >= 0) view.operation.tools.splice(i, 1);
return;
}
}
}
```
Plain mutation, no Immer, no return value. An event for a tool the view has never
seen simply does nothing; the host sends a `replace`.
A lane facet runs this same function under the tracker, so the facet's own ops
fall out against **its** shape — which need not be `LaneView` and generally is
not. The facet never writes an op.
## 8. Wire and consumer
```jsonc
{ "seq": 0, "ops": [["r",{"transcript":[],"operation":null}]] }
{ "seq": 1, "ops": [ … ] }
```
One shape: a batch of ops. The first is a base batch — it begins with `r` —
and everything after is deltas. A gap, a reconnect, a provider
reload, or a fold that could not apply all take one path: send a fresh `replace`.
The consumer is `apply` — six verbs, no domain knowledge, no library, no tool
code, against a plain mutable object it owns.
## 9. What this run costs
256 KB of output, ~20 flushes, 50 KB window:
| | today | this design |
|---|---|---|
| durable writes | full `AgentToolResult` per checkpoint | structural ops plus explicit periodic capped base batches |
| durable location | main log, forever | sidecar, unlinked on settle |
| wire per flush | whole snapshot | one `truncate` + one `append` |
| details written | rebuilt whole, every flush | one `set`, once |
| truncation logic | in `bash.ts` | in the exec env, shared by all tools |
| spill location | `/tmp`, OS-cleaned | exec env, session-scoped |
The details row is the one worth dwelling on. Nothing about bash's details
changed — they were always small. What changed is that they stopped riding inside
a container replaced wholesale on every update, which is why no tool anywhere
currently bothers to mutate them incrementally.

462
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/harness-tools.md vendored
View File

@@ -1,462 +0,0 @@
# Tool Output and Progress
> **Scope:** harness-local. Nothing here depends on the facet system, RPC, or any
> presentation. Depends on [delta tracking](../01-delta/delta.md) (the landed Chord op vocabulary and tracker), [execution environments](../03-execenv/execenv.md) (where truncation happens), and [scoped storage](../02-scopes/scopes.md) (durability).
## 1. The problem
Three faults, one cause.
**Tools each implement their own truncation.** `bash.ts` owns a rolling buffer,
`truncateTail`, a spill file, an update throttle and a checkpoint interval.
`read.ts` has its own truncation. Every future tool producing bulk output
reimplements this, differently.
**Progress is a whole value.** `onUpdate(partialResult)` hands over a complete
`AgentToolResult` on every update, `tool_update` carries that whole result, and
`openToolProgress` persists it with `setValue`.
**`details: unknown` was believed to force this** — you cannot append to a value
whose shape you do not know. That premise is now false. A structural tracker
(`delta.md`) records ops against JSON without knowing the type, so details need
no special handling at all.
## 2. `ToolOutput`
The harness constructs a sink per invocation and passes it to `execute`. **Tools
return nothing**; the sink holds the result.
```ts
interface ToolOutput<TDetails extends JsonValue> {
/** Append to the text block. */
write(text: string): void;
/** Append an image block. Images are never windowed. */
image(image: ImageContent): void;
/** Replace the retained text wholesale. Chord still recovers a verified `t` + `a` slide when possible. */
replace(text: string): void;
/** Apply source-owned truncation totals and spill metadata without resending text. */
capture(metadata: ShellOutputMetadata): void;
/** The tool's own details object. Mutate it directly. */
readonly details: TDetails;
/** Accumulates. A subagent making several model calls adds to it. */
usage(usage: Usage): void;
/** Replaces. */
addTools(names: string[]): void;
/** Replaces — a tool that decides to terminate and then recovers can say so. */
terminate(value: boolean): void;
}
```
```ts
execute(
toolCallId: string,
params: Static<TParameters>,
signal: AbortSignal,
out: ToolOutput<TDetails>,
context: Context,
): Promise<void>;
```
`execute` returns `void`. Everything the tool produces flows through the sink —
including `usage` and `addTools`, which cannot be settle-time return values
because a replayed tool must be able to seed from durable state (§7.4).
**Failure is a thrown error.** `isError` is set by the harness when `execute`
rejects. A tool may throw anything, including errors from libraries it did not
write.
**`terminate` is orthogonal to how execution ended**, so a failing tool can ask
the loop to stop:
```ts
try { await thing(); } catch (error) { out.terminate(true); throw error; }
```
This closes a gap in the current implementation, where `executeToolCall`'s catch
hardcodes `isError: true` with no terminate, and `immediateError`'s terminate
parameter is only ever passed `true` by `applyBeforeToolDecision` for a hook
block.
### 2.1 Details are just an object
```ts
async execute(id, params, signal, out, context) {
out.details.total = 42;
out.details.passed += 1;
out.details.failures.push({ name, message });
out.details.current = undefined; // -> delete
}
```
Fully granular ops fall out, verified against the prototype:
```jsonc
["#",0,["details","passed"]]
["s",0,1]
["p",["details","failures"],0,0,[{…}]]
["d",["details","current"]]
```
No recipes, no mutation map, no `initialDetails` on `tool_start`, no Immer, and
no consumer ever runs tool code. `TDetails` remains the tool's own exported type
so a renderer can cast `call.details` to it — that is all it is for.
A survey of `packages/agent/src/harness/tools/` found **no tool mutating details
incrementally today**; only `bash.ts` writes them mid-stream and it rebuilds them
whole, because the container was replaced wholesale anyway. This design removes
that cause, and costs nothing if nobody takes it up.
**Caveat:** details are now unbounded. A tool pushing to `failures` in a loop
grows without limit, and unlike text there is no cap. Nothing shipped does this;
the door is open in a way it was not when details were a replace-only value.
### 2.2 Partial output survives failure
Today `executeToolCall` catches and returns `createErrorToolResult(message)`,
building a fresh result from the error string alone — a tool that streamed 8 KB
and then threw reports only the error.
With the sink, what was written is already the result. The harness appends the
error text as content and keeps the rest. The error text is model-visible
content, not presentation: the model needs to read why the call failed.
`abortedMessage` and `interruptedMessage` should be made consistent with this;
today they build replacement results via `syntheticMessage`, so a cancelled
long-running command loses its partial output too.
## 3. Content
Content is exactly **one text block followed by zero or more image blocks**. A
tool cannot interleave text between images — text written before and after an
image lands in the same block. Deliberate: truncation only ever touches a string,
and an image is never partially anything.
Images are **never windowed**. A byte or line cap over base64 payloads is
meaningless and `truncateTail` operates on text. `maxBytes` / `maxLines` govern
text only.
### 3.1 Retention mode
Declared on the tool definition:
```ts
output?: {
retain?: "head" | "tail"; // default "tail"
maxBytes?: number;
maxLines?: number;
}
```
**`head`** — append until the cap, then stop. Nothing is ever removed. Right for
output meaningful from the start: file reads, listings, greps.
**`tail`** — a rolling window. Right for anything whose interesting part is at
the end: builds, test runs.
Head+tail is **not** offered. `truncate.ts` exports `truncateHead` and
`truncateTail`; neither combines them and none is being added.
### 3.2 The tool does not truncate — the exec env does
For output originating in the execution environment, capping, coalescing and
spilling happen **at the source**. See [`execenv.md`](../03-execenv/execenv.md). The tool passes its
`ShellOutputLimits` into `env.exec` and pipes the resulting updates into the sink.
This is not a convenience. Reading a 1 GB file on a sandbox host must not ship
1 GB to the agent machine to be capped there, and the spill file must land where
the model's own `read` and `grep` run.
For output originating on the agent machine (subagents, in-process work),
`ToolOutput` applies the same logic locally. Same code, different location.
Note this reverses an earlier decision that there is no spill in the sink and a
temp-file path is tool-specific. The exec-env argument — model reachability
across a machine boundary — is what changed it.
## 4. Tool output state
`tool_update` carries `Op[]` (`delta.md` §6) targeting the invocation's
`ToolOutputState`:
```ts
interface ToolOutputState {
content: (TextContent | ImageContent)[];
details: JsonValue;
usage?: Usage;
addedTools?: string[];
terminate: boolean;
truncation: ShellOutputTruncation; // totals over everything ever written, without duplicate text
}
```
Ops rather than a typed variant union, for one decisive reason: **only ops give
details granularity without the harness knowing `TDetails`**. A typed union would
need per-tool recipes, which is the machinery §2.1 deletes.
Text still gets delta treatment, because the sink applies the cap *before*
mutating, so a rolling window produces `truncate` + `append` on
`content[0].text` rather than a whole-value set.
Interned ops also measure **smaller than a typed frame vocabulary** on every
workload, and 10x smaller on details, because a frame repeats `toolCallId` where
an interned op carries one integer. See `delta.md` §4.1.
There is no `drop` event. The sink knows what it evicted and expresses it as
`truncate`; a consumer needs no separate signal and derives nothing.
## 5. Harness events
```ts
| { type: "tool_start";
runId; turnId; toolCallId; toolName;
args: unknown }
| { type: "tool_update";
runId; turnId; toolCallId;
ops: Op[] } // a base batch begins with `r`
| { type: "tool_end";
runId; turnId; toolCallId;
isError: boolean }
```
**`tool_start` carries identity only.** An earlier draft added `caps` and
`initial`; both were redundant and are gone. The first batch is always a base batch
(`delta.md` §6), so the initial state arrives on the update channel — carrying it
twice means two ways to establish the base, which can disagree. And `caps` are
already inside the state: `ToolOutputState.truncation` carries `maxBytes` and
`maxLines`, which is what a renderer needs to say "50 KB limit". The draft itself
conceded a consumer "no longer has to" fold identically, since the producer's ops
encode eviction — that was the last reason for `caps` to travel, and it does not
hold.
`tool_update` carries ops. A base batch travels the same channel as a delta,
because a replacement is itself an op (`delta.md` §2) — there is no second shape. `message_update` has the identical shape
(`message-update.md` §5.1); a consumer folds tool output and assistant output with
one code path.
**`tool_end` carries no content, no details, no usage, no terminate.** Every byte
already went out. Re-sending would duplicate each base64 image at the moment
nothing new has happened.
> **The fold is the result.** Nothing a consumer folded is ever re-sent to
> confirm it.
This dissolves an earlier open question about settle-time truncation disagreeing
with the running fold. There is no separate settle-time truncation: the sink's
window *is* the truncation. Anything a tool wants to add at the end — bash's
`[Showing lines 8000-8123 of 8123]` footer — is `out.write(footer)`, one more
append.
The harness still assembles `AgentToolResult` in process for the model, and
`createToolResultMessage` still writes the settled `ToolResultMessage` to the
transcript with `content`, `details`, `usage`, `addedToolNames`, `isError`.
Neither is a wire event.
## 6. Lane reduction
```ts
export function reduceLaneSnapshot(view: LaneView, event: HarnessEvent): void;
```
Plain mutation on a plain object. No `Draft`, no `produce`, no Immer, and **no
`Rebase` return value** — a fold that cannot apply an event leaves state alone,
and the host sends a `replace`. (An earlier draft had `void | Rebase`; under
Immer that throws, and without Immer there is nothing to return to. Note
`reducer.ts:4` currently defines `LaneSnapshotReduction = LaneSnapshot | { rebase: true }`,
so this is a real change to existing code, and `navigation_end` is the event that
drives it.)
The event is the only input besides the view. No registry, no `resolve`, no tool
code — so a tool rewritten between crash and resume cannot make a persisted
stream unreadable.
When the harness is wrapped by a facet, the same mutation runs under the tracker
from `delta.md` and ops fall out. The harness itself does not know this.
## 7. Durability
### 7.1 What exists today
`pendingToolOutput(operationId, invocationId)` is a
`value<AgentToolResult<unknown>>`, and — importantly — `progress.write(partial)`
fires **only** when `options?.checkpoint === true` (`drive/tools.ts:325`). Not on
every update. bash checkpoints every 2 s with a `JSON.stringify` dedupe; every
other tool never checkpoints at all.
It is also **not a progress buffer**. It is the interruption checkpoint. On
resume, if the tool is not replay-safe, `readCheckpoint` turns it into a real
`ToolResultMessage` for the model: `[...checkpoint.content, INTERRUPTION_MARKER]`
plus `details` and `usage`.
Note the implication that misled an earlier draft: because the checkpoint must
*be* the current state, it was read as needing `value` semantics. It does not. It
needs to be *derivable*, and folding encoded batches from the last base batch derives it — which is the point of tagging base batches (§7.3). `checkpoint: true`
requests a durable write, not a replacement.
`pendingAssistantFrames` is a `list<AssistantMessageFrame>` appended per frame,
`deleteList` on settle in `response.ts:345`, `deferred.ts:157`, `terminal.ts:47`.
Note `operationCleanupWrites` (`terminal.ts:26`) does four `scanValues` calls to
enumerate what to delete at settle. Under scopes, the two covering
`operationToolMemoPrefix` and `pendingToolOutputPrefix` are replaced by a single
`retireScope(operationId)`.
### 7.2 Renaming
`pendingAssistantFrames` → **`pendingAssistantOutput`**, matching
`pendingToolOutput`. Frames stop being the durable unit; both addresses now hold
tracked state written as ops or snapshots.
### 7.3 What to write, and when
Both addresses are **ephemeral-scoped** ([scopes.md](../02-scopes/scopes.md)), so they live in a
sidecar retired on settle rather than in the main log forever.
Two independent wins, in order of importance:
- **Encoding.** Ops instead of whole values, plus address interning: 93.89 MB to
5.32 MB in a single file, no change to atomicity. Do this first.
- **Scopes.** Pending state leaves the main log entirely: 5.32 MB to 0.06 MB
surviving.
**Both are `list<WireOp[]>`**, not values. The sink owns one stateful Chord encoder/decoder pair per durable value stream. One encoded batch is appended per flush, and a logical batch whose first op is `r` is tagged `"base"` on the storage record.
Recovery reads backwards with `stopAtTag: "base"` and applies forward
([delta.md §9](../01-delta/delta.md#9-durable-form), [scopes.md §11](../02-scopes/scopes.md#11-list-tags-and-stop-conditions)).
Writing one flush:
```ts
const ops = out.flush();
if (ops.length === 0) return;
const wire = enc.encode(ops);
writes: [appendList(address, wire, isBase(ops) ? "base" : undefined)];
```
`isBase` comes from Chord. It checks for the root replacement op `r`; ordinary nested sets use `s`. Classification lives beside the vocabulary so the comparison is written once.
The landed tracker emits structural ops unconditionally. There is no serialized-size comparison or adaptive replacement heuristic. A producer requests a base batch explicitly with `rebase()`; the output sink's cap bounds that replacement, while periodic rebasing bounds recovery work. Production remeasurement rejects a text-specific append/truncate API: the generic path measured 2.43–2.46 µs per 50 KB rolling-window flush locally, below surrounding costs. Keep ordinary tracked string mutation; see the [decision record](../01-delta/append-decision.md).
**Checkpointing is not deleted.** `BASH_CHECKPOINT_INTERVAL_MS` remains only as an interim compatibility mechanism. The generic sink owns durable frequency because Shell cannot price a storage write or enforce memo/output atomicity. Forced memo, terminal, and recovery-base writes bypass ordinary pacing while remaining cap-bounded.
### 7.4 Replay must seed, not discard
`clearReplayCheckpoint` currently writes `deleteValue(pendingToolOutput(...))`
before re-executing a replay-safe tool (`drive/tools.ts:257`). **This is a bug.**
Replay-safe means the tool is re-executed, but memos exist precisely so it does
*not* redo work it already did — and skipped work emits nothing. Any output for
memoised work is lost today.
The fix: seed a fresh `ToolOutput` from the durable state, then re-execute. The
tool appends to a sink that already holds what it produced before the crash.
This is also why nothing may be settle-only. `usage` and `addTools` must survive
a seed, so they flow through the sink like everything else.
### 7.5 The memo invariant
> A tool's memo write and its output checkpoint must commit in the same
> transaction.
Otherwise a tool does work, sets a memo, crashes before the next checkpoint, and
on replay skips the work while the seeded output has no record of it.
**This does not hold today.** `setMemo` (`drive/tools.ts:112`) and `openProgress`
(`runtime/progress.ts:44`) are two separate `lane.command` calls, hence two
transactions. Making it hold means bundling the checkpoint into the memo's commit:
```ts
setMemo(name, value) {
validateMemoName(name);
if (!active) return Promise.reject(ended());
return lane.command<void>((state) => {
if (!ownsEffect(state)) return { kind: "reject", error: ended() };
const memo = operationToolMemo(drive.operationId, call.resultEntryId, name);
return {
kind: "commit",
writes: [
value === undefined ? deleteValue(memo) : setValue(memo, value),
setValue(pendingToolOutput(drive.operationId, call.resultEntryId), out.snapshot()),
],
next: state,
materialize: () => undefined,
};
}, drive.context);
}
```
Both addresses are **ephemeral-scoped**, so this is a single-file transaction and
statically enforced as one ([scopes.md §3 and §6](../02-scopes/scopes.md)). `operationToolMemo` is
placed in that scope precisely for this. Ordering on the Session line would not be
enough — two file writes are not atomic. The periodic checkpoint stays as it
is — best-effort, for the interruption path; this forces one where correctness
requires it.
The invariant holds because the tool does X, writes X's output to the sink, *then*
calls `setMemo("did X")` — so the sink's state at commit time already contains X's
output.
**Ordering alone does not work**, in case it looks tempting:
- memo first, checkpoint second → crash between → replay skips X and the seeded
output lacks it → silent loss;
- checkpoint first, memo second → crash between → replay redoes X and appends
again → duplicated output.
Duplication is the less bad failure, so ordered writes are a tolerable fallback,
but neither is correct.
**Cost:** `setMemo` now writes a full capped output state rather than a small
value. Memos are rare — a handful per invocation — so this is bounded by
`memo count x cap`, not by output volume.
### 7.6 `openProgress` has a write-ordering race
`commitWrite(item)` captures `item` when `write()` is called, and the write is
fire-and-forget with only `latest` tracked. A checkpoint captured at T1 can
therefore commit *after* a memo-bundled checkpoint at T2, overwriting newer state
with older — reintroducing exactly the loss §7.5 prevents.
Fix: resolve the sink's state inside the command planner rather than at call time.
```ts
commitWrite: () => setValue(address, out.snapshot()) // evaluated under the Session line
```
`lane.command` serializes on the Session line, so checkpoint writes become
monotonic by construction. This removes the race generally, not only against
memos.
### 7.7 Nothing on the durable path runs tool code
Ops are interpreted by a six-verb applier with no domain knowledge, so a tool
rewritten between crash and resume cannot make a persisted stream unreadable.
> **The durable path uses only harness-owned reducers.**
This also rules out persisting *facet* ops. The harness has no facet state,
facets come and go, and a recovering harness must rebuild its working values with
no facet present.
## 8. Open questions
- **Property tests for the tracker** (`delta.md` §3.3). Everything here rests on
producer and replica agreeing; nothing currently proves they do.
- Coalescing window: per tick, or a byte/time threshold.
- Whether image count needs a bound. Images are unwindowed, so a tool pushing
them in a loop grows `content` without limit. Currently treated as a tool bug.
- Whether details need a bound for the same reason (§2.1).
- Whether `retain: "head"` should keep emitting counter-only updates once capped
so a renderer can report how much was suppressed. [`execenv.md`](../03-execenv/execenv.md) answers this
for exec-originated output; agent-side output needs the same answer.
- Whether a failing tool *should* be able to terminate, or whether the current
inability is deliberate — there is a reasonable argument the model should
receive the error and decide.
- **Derived values.** `arguments` parsed from accumulated JSON should not be
replicated; derive it on demand. Safe because `parseStreamingJson` is total —
four fallbacks ending in `{}`, it cannot throw — so a replica derives with no
error path and no agreement protocol. Generalises to: no derived fields in
replicated state.

125
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/04-tool-output/rate-limiting.md vendored
View File

@@ -1,125 +0,0 @@
# Bounded output publication
**Status:** the shared adaptive publisher and its `ExecutionEnv` use are implemented. Generic `ToolOutput` integration remains design work.
Depends on landed Chord delta tracking, source-bounded execution output, and scoped storage for durable batches.
## 1. Invariant
> The durable record, what the model sees, and what the UI shows are the same bounded view.
A spill file is not a second view. It is a file inside the execution environment that the model reaches through ordinary file tools.
Every uncontrolled producer boundary needs two independent bounds:
- **state size:** the latest retained text is capped;
- **publication:** both encoded bytes and event count are paced.
Delta encoding is complementary. It compresses a published change; it does not cap state or decide when publication occurs.
## 2. Boundaries
```text
remote process
-> optional ExecutionEnv publisher
-> worker ToolOutput publisher
-> events + durability + replication
```
A publisher instance is required only before a real costly boundary:
- a physically remote execution environment limits output before transport;
- `ToolOutput` limits custom tools and downstream event/storage traffic;
- a colocated environment can feed its bounded updates in process without another serialized transport;
- custom tools bypass `ExecutionEnv` but cannot bypass `ToolOutput`.
The control algorithm is shared. Payload vocabularies differ: Shell uses replace/append/slide/metadata; `ToolOutput` uses Chord operations.
## 3. Why size or cadence alone is insufficient
A fixed 50 KB snapshot is safe per event but not over time. At 100 ms it permits ten complete snapshots per second, approximately 500 KB/s plus envelopes.
A byte budget alone also permits excessive tiny events and durable transactions. A minimum interval bounds count; encoded-size debt bounds bandwidth.
The original handoff incorrectly treated `intervalMs = 100` as 100 emits/s. It is ten emits/s. The fixed interval was still non-adaptive: it delayed small trickles while allowing complete windows at the same frequency.
## 4. Landed adaptive algorithm
`packages/agent/src/harness/utils/adaptive-publisher.ts` implements:
```ts
nextDelayMs = max(globalMinEmitInterval, encodedUpdateBytes * 1000 / globalTargetBytesPerSecond);
```
Current harness-global policy:
```ts
minEmitInterval = 100 ms;
targetBytesPerSecond = 100 KB/s;
```
Behavior:
1. The first dirty state after idle publishes immediately.
2. Writes before the next deadline collapse into the latest state.
3. One trailing timer publishes held state after the deadline.
4. Completion and correctness boundaries force one bounded publication.
5. The publisher commits its baseline before consumer delivery, preventing duplicate deltas if a consumer applies and then throws.
This is an amortized token bucket with a cap-sized burst. A leading or forced terminal update may exceed the target over a short interval, but sustained encoded bytes converge to the target and sustained event count cannot exceed the minimum-interval floor except for explicit forced correctness writes.
## 5. Scenario traces
Assume a 50 KB cap, 100 KB/s target, and 100 ms floor.
### Below cap, completes
The initial state publishes immediately. Small appends publish no faster than the floor, and final dirty state is forced. Total encoded text is approximately the produced text.
### Below cap, trickling
Writes arriving more than 100 ms apart publish immediately because the preceding deadline has already passed. Faster writes collapse into one append per floor interval.
### Above cap, full force
The retained state never exceeds 50 KB. Complete window turnovers encode as bounded replacements. A roughly 50 KB update buys about 500 ms of silence, yielding approximately two updates and 100 KB per second regardless of raw producer throughput.
For shell execution, the complete stream goes to a source-local spill under backpressure rather than over the output-update channel.
### Above cap, trickling
A small tail movement encodes as truncate plus append (Chord) or `slide` (Shell). Its encoded size is small, so the 100 ms floor dominates and output remains responsive. Resending a complete 50 KB snapshot for each tiny slide would be both slower and larger.
### Burst then silence
The leading state is immediate. Held writes collapse, and one trailing timer publishes the latest residue. There is no polling timer.
### Huge single write
The producer may already have allocated its input, but the boundary retains and publishes only the configured cap. Shell spill keeps the complete source stream. Arbitrary custom-tool images and structured details still require separate limits.
## 6. Forced writes
These bypass pacing once, while remaining state-size bounded:
- command/tool completion;
- error or abort;
- a memo and output checkpoint committed atomically;
- an explicit recovery base/rebase.
A terminal flush cancels the trailing timer before `tool_end`, preventing a late update for a settled invocation.
## 7. `ToolOutput` application
The sink will own one Chord tracker and encoder per invocation. Mutations remain local while publication is blocked; flush-time dirty tracking means held writes collapse without retaining one op per write.
One sink flush feeds the live event and current model-visible state. Durable batches use the same logical flush, encoded as per-stream `WireOp[]`. Periodic `rebase()` bounds recovery replay; a memo correctness flush writes a tagged base batch in the same transaction as the memo.
Durable cadence belongs to the sink, not to Shell or bash. Bash's existing two-second checkpoint request remains only as an interim compatibility mechanism until the sink migration.
## 8. Remaining decisions
- Explicit byte/count rejection policy for images.
- Bounds for structured details; arbitrary JSON cannot be meaningfully tail-windowed.
- Whether ordinary durable writes ride every live sink publication initially or use a slower measured cadence. Memo and terminal correctness flushes are not optional.
- Exact global production values after profiling real remote transport and storage.

227
third_party/pi/packages/agent/docs/mobile-handoff/01-harness/05-assistant-output/message-update.md vendored
View File

@@ -1,227 +0,0 @@
# `message_update` Write Amplification
> **Scope:** harness-local. Depends on [delta tracking](../01-delta/delta.md) for the landed Chord `Op`/`WireOp` vocabulary and [scoped storage](../02-scopes/scopes.md) for durability. Chord delta tracking has landed; scoped storage and this Harness integration have not.
## 1. The problem
```ts
{ type: "message_update", runId, message, event, frame? }
```
Three representations of the same thing travel together:
- `message` — the full `AssistantMessage`;
- `event` — an `AssistantMessageEvent`, which itself carries `partial: AssistantMessage`,
a second full copy;
- `frame` — the actual delta, optional.
Per streamed token this is roughly two complete snapshots plus a delta, so bytes grow
quadratically over a response.
The reducer does not even use the delta:
```ts
case "message_update":
if (next.operation?.id === event.runId && event.message.role === "assistant") {
next.operation.streamingMessage = event.message;
}
```
A straight assignment. So on any wire, sending the event is *worse* than sending a
snapshot — it ships two where `replace` ships one.
The wire adapter is already halfway to the fix: it drops `event` and sends `message`
plus `frame`. That is the full snapshot together with the delta that produced it.
## 2. The compact form already exists
```ts
/**
* Compact, replayable assistant-message progress. Terminal settlement is
* intentionally excluded and must be persisted separately.
*/
export type AssistantMessageFrame =
| { type: "start"; partial: AssistantMessage }
| { type: "text_start" | "text_delta" | "text_end"; contentIndex: number; ... }
| { type: "thinking_start" | "thinking_delta" | "thinking_end"; ... }
| { type: "toolcall_start" | "toolcall_checkpoint" | "toolcall_delta" | "toolcall_end"; ... };
```
`AssistantMessageFrameEncoder` produces it, `reduceAssistantMessageFrames` folds it,
and `openFrameProgress` already persists it with `appendList`. The delta format is
built, used, and durable. It simply is not what `message_update` carries.
## 3. Precedent
pi already does wire-is-delta one layer down. `PiMessagesEvent` — the serialized form
a pi-messages backend sends — has no `partial`:
```ts
| { type: "text_delta"; contentIndex: number; delta: string }
| { type: "text_end"; contentIndex: number; content: string; contentSignature?: string }
```
`pi-messages.ts` then rehydrates: it holds a local `partial`, mutates it per event
(`partial.content[i].text += event.delta`), and returns the in-process
`AssistantMessageEvent` with `partial` attached.
So the convention is established. It just stops at the provider edge instead of
continuing through the harness.
## 4. The change
```ts
| { type: "message_update"; runId: string; entryId: string; frame: AssistantMessageFrame }
```
`message` and `event` are removed; `frame` stops being optional.
The blast radius is small because `HarnessEvent` is not what most streaming consumers
read. `AgentEvent` (`agent-loop.ts`) and `AgentSessionEvent` (`agent-session.ts`) are
separate unions that happen to share the tag name and build their own
`message_update` from the pi-ai event directly. They are out of scope here.
Real consumers and producers of `HarnessEvent.message_update`:
| site | change |
| --- | --- |
| `runtime/drive/response.ts` | emit the required semantic frame; stop attaching full snapshots |
| `runtime/reducer.ts` | fold the frame instead of assigning `event.message` |
| lane/facet state adapter | run that fold under the Chord tracker and emit `Op[]`/encoded `WireOp[]` |
| `experimental/harness-wire-adapter.ts` | stop treating raw `HarnessEvent` as the final replication format |
| `harness/telemetry.ts` | name list only |
| `protocol/harness.ts` | replication carries encoded `WireOp[]`, not the raw event |
`message_end` continues to carry the settled message, because frames deliberately
exclude terminal settlement. That is once per message, not once per token.
## 5. An incremental applier is needed
`reduceAssistantMessageFrames` is a whole-stream fold over an `Iterable`. The
reducer needs a step function:
```ts
export function applyAssistantMessageFrame(
state: AssistantFrameState,
frame: AssistantMessageFrame,
): void;
```
Plain mutation on a plain object. **No `Draft`, no Immer.** An earlier draft
argued for a draft-mutating signature so it would compose inside a `produce`
recipe; that motivation is gone ([delta.md §8](../01-delta/delta.md#8-what-this-removes-from-the-codebase)). The step function is still
needed — the whole-stream version becomes a loop over it — just for the simpler
reason that the reducer folds one frame at a time.
### 5.1 pi-ai frames stay at the pi-ai boundary; Chord ops cross replication boundaries
`AssistantMessageFrame` is pi-ai's semantic delta vocabulary (`text_delta`, `text_end`, …). [Delta tracking §6](../01-delta/delta.md#6-there-is-no-frame-type) defines no second frame wrapper: in-process replication carries `Op[]`, and a wire adapter carries encoded `WireOp[]`. Pi-ai frames stop at the fold; Chord ops cross the replication boundary.
`AssistantMessageFrame` is pi-ai's own delta vocabulary and stays. What changes is
that it is no longer the durable unit or the replication unit.
The harness folds frames into `LaneView` by plain mutation. Under the Chord tracker that yields:
```json
["a",["operation","streamingMessage","content",0,"text"],"Let me "]
```
Measured, interned ops are **smaller than frames** on this workload — 13.6 KB
against 21.5 KB raw for 200 deltas — because a frame carries three keys of its own
where an interned op carries one integer. So the size argument for putting frames
on the wire is dead.
What frames keep is semantic: `text_end` carries authoritative content plus a
signature in one atomic unit, where ops would need two with a weaker contract
about their relationship. That is why they remain the *input* vocabulary and are
folded before anything crosses a boundary.
### 5.2 Reducer state must live in the reduced value
Text and thinking fold purely: `block.text += frame.delta`, and `*_end` overwrites
with authoritative content plus signature. The `ended` flag in `ReducerBlockState`
is used only for validation and can be dropped.
Tool calls cannot. `toolcall_delta` does `state.json += frame.delta`, accumulating
a **raw JSON string that is never stored on the message** — the block holds
`arguments`, the parsed value. You cannot recover the accumulator from the
snapshot, and you cannot append a delta to a parsed object.
So the accumulator has to become part of the value being folded — e.g.
`operation.frameState[contentIndex].json` on `LaneView` — leaving
`AssistantMessage` clean. Generally:
> **A replicated reducer's state must be part of the replicated value.** Anything
> held beside it diverges on any consumer that did not run the producer's fold.
The corollary is that `arguments` itself should **not** be replicated. It is
derived from `json`, and a parsed object is a fresh reference on every parse, so
storing both ships two copies of the same information. Derive it on demand.
This is safe precisely because `parseStreamingJson` is **total** — four fallbacks
ending in `{}`, it cannot throw — so a replica derives unconditionally with no
error path and no agreement protocol. There is no parse-failure state to
represent, and no block needs an error slot.
### 5.3 Parse cost
`parseStreamingJson` on a growing string once per delta is quadratic per message.
Since `arguments` is now derived rather than replicated, this cost falls on whoever reads it rather than on every consumer. Presentation can refresh derived arguments at semantic checkpoints and `toolcall_end` instead of parsing on every delta; that policy is separate from why the encoder currently emits a checkpoint (§6).
## 6. What `toolcall_checkpoint` is for
`EncoderBlockState` carries `caughtUp` and `catchupJson` because a queued provider event's shared `partial` may already be ahead of that event's delta. The checkpoint catches the semantic frame stream up to the authoritative tool-call arguments visible at block start; it is not currently a general late-subscriber protocol.
After frames fold into tracked state, a Chord root replacement (`r`) is the replication and durable-recovery resync point. `toolcall_checkpoint` remains semantic input to that fold rather than carrying transport responsibility.
## 7. Write volume for pending output
`openFrameProgress` calls `appendList(pendingAssistantFrames(...))`, so every
frame is a line. A long response is thousands of writes.
**The address is renamed `pendingAssistantOutput`** and becomes a `list<WireOp[]>`, matching `pendingToolOutput` ([tool-output handoff §7.2](../04-tool-output/harness-tools.md#72-renaming)), and it stops being a list of frames. The progress sink encodes tracked `Op[]` with one stateful encoder per response before appending each durable `WireOp[]` batch; explicit `rebase()` calls produce bounded root-replacement batches for recovery. The list lives in an **ephemeral scope** so it is unlinked on settle rather than persisting in the main log ([scoped storage](../02-scopes/scopes.md)).
The important property is that this list is **not history**: `deleteList` runs on
settle in `response.ts`, `deferred.ts`, and `terminal.ts`. It exists so a crash
mid-response can recover a partial assistant message. The settled message is persisted
separately.
That means per-frame durability buys almost nothing, and the write rate can be traded
away directly:
- **Coalesce with a bounded, non-resetting window.** The first pending frame opens the
window; later frames join it *without* extending the deadline, so a continuously
streaming response cannot postpone the first write indefinitely — the failure mode a
naive debounce has. Frames admitted during an active write form the next batch.
- **Concatenate on flush.** A run of `text_delta` frames for one `contentIndex`
collapses to a single frame with the joined text. The fold result is identical, so
this is lossless for our purposes.
A crash then loses at most one window of in-flight streaming.
### 7.1 Contrast with DeepSeek Harness
DSH cannot make this trade. Its `assistant/chunk` events are canonical log entries, so
per-token durability is mandatory and it attacks size instead:
- the same bounded non-resetting write-behind window;
- **packed rows** — runs of consecutive chunk deltas stored as
`text-chunks` / `reasoning-chunks` / `tool-call-chunks`, lossless and roughly 60%
smaller on a real session, with reading unconditional so layout never depends on the
write switch;
- checksummed zstd frames by default, with recovery from a torn final frame.
Their packing must reconstruct exact event boundaries, sequence numbers and timestamps,
because `seq = log.length` and validation requires a contiguous logical log. Ours does
not, because the frames are discarded at settle — so plain concatenation is available
to us and packing machinery is not needed.
## 8. Consequence for ad-hoc listeners
`HarnessEvent.message_update` stops being self-describing. A listener that attaches
mid-stream sees a delta against a partial it does not hold.
Any correct listener already has the base, because `watch()` installs the subscription
and captures the snapshot inside one `readLane` critical section, buffering until
`start()`. But a listener that calls `on("message_update")` directly, without a watch,
is no longer viable — worth knowing before committing.

1855
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/01-facets/facets.md vendored
View File

File diff suppressed because it is too large Load Diff

162
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/README.md vendored
View File

@@ -1,162 +0,0 @@
# Facet sandbox — isolated-vm
**Unmodified pi facet code running in a V8 isolate with no ambient authority.**
Full JIT, so ~1.8× native rather than the 8–17× a WASM interpreter costs.
```bash
npm install
npm run demo # a facet: contributions, components, commands, callbacks
npm run audit # escape audit — the interesting one
npm run bench # crossing cost, isolate cost, 300 components
npm test # 412 property assertions
```
Node 22+ (`--experimental-strip-types`). See `../facet-sandbox-poc` for the
QuickJS-WASM equivalent.
## The safety claim, and why it is structural
isolated-vm **can** be used unsafely, and that is what `facets.md` §14.2 objected
to. The footgun is handing the guest a live handle to a host object — via
`derefInto()`, or by returning a `Reference` from a host call — after which the
guest walks that object's prototype chain into the host realm.
This membrane makes that impossible **by construction, not by care**:
1. `encode()` is the only way a host value reaches the guest, and it is
`JSON.stringify` with a replacer. **Its output is a string.** A string cannot
carry a reference.
2. Host callables never cross. They become an integer id into `hostTable`, which
lives entirely host-side. The guest receives a **number**.
3. Exactly **one** `Reference` is ever given to the guest — `__invokeRef`, the
single call-in point.
4. `derefInto()` is used exactly once, on the guest's **own** global. No host
object is ever its argument.
The guest's entire view of the host is `{ number, string }`. There is no object
graph, so there is nothing to walk.
### Audit results (`npm run audit`)
```
Ambient authority:
require / process / fetch ["undefined","undefined","undefined"]
global names visible 64 globals
Classic escape ladders:
Function('return process')() undefined
constructor walk on host data undefined
The isolated-vm Reference footgun:
__invokeRef.deref() blocked: TypeError ("Cannot dereference
this from current isolate")
__invokeRef.copySync() blocked: TypeError
__invokeRef.getSync('constructor') blocked: TypeError
derefInto() is callable? inert (object)
invoke derefInto() result blocked: TypeError
host globals via derefInto() no host globals
Resource bounds:
spinning guest interrupted after 205ms
isolate usable afterwards true
```
One probe gave a **false positive** in a first draft: testing `f.deref === undefined`
to prove a returned host function is a Proxy rather than a Reference. The
membrane's proxy returns a proxy for *every* property, so `f.deref` is truthy.
That is not a leak — calling it routes to `hostFn["deref"]`, which does not exist
and throws host-side. The audit now calls it instead of checking for absence.
## Performance
| | isolated-vm | QuickJS | native |
| --- | --- | --- | --- |
| 300 markdown components, small | **70 ms** | 494 ms | ~40 ms |
| 300 markdown components, typical | **108 ms** | 1118 ms | 64 ms |
| 300 markdown components, large | **419 ms** | 4260 ms | ~150 ms |
| 146 KB bundle load | **35 ms** | 104 ms | — |
| membrane crossing | **4.5 µs** | 7–17 µs | — |
| per compartment | 1080 KB, 5.5 ms | **77 KB, 0.8 ms** | — |
**~1.8× native at typical size.** QuickJS is 8–17× and gets worse with input size.
Isolates cost **14× more memory than a QuickJS runtime** (1080 KB vs 77 KB) and
are slower to create. For a handful of facets that is irrelevant; for hundreds it
would not be.
`Intl.Segmenter` **exists** here, which is the single hard blocker for reusing
`packages/tui` components under QuickJS. That alone may decide it.
## Limitations
**Budgets do not nest.** A timeout bounds one `evalSync`. A guest function
re-entered *by the host* — a contributed callback, a component method — gets
`callGuestRef`'s own budget, not the outer one. Asserted explicitly in
`property-test.ts`; a runaway callback took 5005 ms under an outer budget of 50 ms.
Bounding total facet time needs separate accounting.
**Async is a settle-callback, not a native promise.** `applySync` is synchronous,
so a host Promise cannot cross. The host returns a token, the guest builds a real
Promise against it, and the host settles it by calling back in. Verified: two
sequential guest `await`s while the host event loop ticked 22 times.
**Native addon, and the ABI matrix is real.** `isolated-vm` ships prebuilds and
only falls back to `node-gyp rebuild` when none matches — which is why install
takes a second, not an hour. It is **not** building V8; V8 is already in the Node
binary. But coverage is narrow:
| version | engines | prebuilds |
| --- | --- | --- |
| **6.2.0** | `>=22.0.0` | linux-x64/arm64, darwin-arm64, win32-x64 — abi127, abi137 |
| 7.0.1 | `>=24.0.0` | — |
| 7.0.0 | `>=26.0.0` | — |
darwin-**x64** is absent everywhere. Installing `isolated-vm@7` on Node 22 falls
through to a source build and **fails without Python and a C++ toolchain** —
verified here. Note also that 6.2.0 is `latest` on npm despite 7.0.1 existing,
because it published one day later.
The project is **not abandonware** — 6.0.1 (Jul 2025) through 7.0.1 (Aug 2026),
actively released. §14.2's "maintenance mode" claim is out of date and should be
corrected. The real risk is different: adopting it couples your minimum Node
version to theirs, and they dropped Node 20 in 6.x and Node 22 in 7.x within a
year. Shipping SEA builds moves that from every user's install to your CI.
**Memory: references release, but lazily.** The `WeakRef` + `FinalizationRegistry`
interning table ports unchanged from the QuickJS membrane and works — 20 000
crossings went to **0 live refs**. But finalization is not prompt: 5 000 refs were
still live 200 ms after a forced GC. Under sustained churn refs accumulate faster
than the collector reclaims them, and a 32 MB isolate did hit its limit.
The consequence is an API rule, not a membrane fix: **do not create references in
a hot path.** `slots.claim(factory)` at construct is one reference forever; a
component method returning a fresh closure every frame is one per frame.
**Hitting `memoryLimit` kills the facet cleanly.** It throws a catchable
`"Isolate was disposed during execution due to memory limit"` and the host
survives — but the isolate is dead and unrecoverable, so teardown must tolerate
an already-disposed isolate. `dispose()` on it throws `"Isolate is already
disposed"`; the membrane guards this.
> **Do not call `isolate.getHeapStatisticsSync()` on a stressed isolate.** It
> **aborted the whole process** during testing — a hard crash, not an exception.
> A `heapMB()` helper was removed from the membrane rather than shipped.
**Still one process, one engine.** A separate isolate is a far stronger boundary
than SES, but Figma's argument for QuickJS was that a *different VM* cannot
confuse objects because the representations differ. Here the guarantee is V8's
isolate boundary plus the membrane's discipline — strong, and structurally
enforced above, but not the same category of claim.
## Files
- `src/membrane.ts` — the membrane. The safety argument is in the header comment.
- `src/facet-example.js` — unmodified facet code: `ctx.use()`, `slots.claim()`
with a closure, a class instance returned to the host, a subscribe callback.
- `src/demo-facet.ts` — loads it and exercises four boundary crossings.
- `src/escape-audit.ts` — the audit above.
- `src/property-test.ts` — 412 assertions: random `JsonValue` round-trips both
directions, identity across crossings, callables, error propagation, reference
release under GC, prototype pollution both ways, budgets, async, disposal.
- `src/bench.ts` — the numbers above.
- `src/markdown-bundle.js` — the real pi `Markdown` component, esbuild-bundled.

13
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/package.json vendored
View File

@@ -1,13 +0,0 @@
{
"name": "facet-sandbox-ivm",
"private": true,
"type": "module",
"description": "Running pi facets in an isolated-vm isolate: full JIT, no ambient authority.",
"scripts": {
"demo": "node --experimental-strip-types src/demo-facet.ts",
"audit": "node --experimental-strip-types src/escape-audit.ts",
"bench": "node --experimental-strip-types src/bench.ts",
"test": "node --experimental-strip-types --expose-gc src/property-test.ts"
},
"dependencies": { "isolated-vm": "6.2.0" }
}

48
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/bench.ts vendored
View File

@@ -1,48 +0,0 @@
/**
* Cost of the isolate, the membrane, and a real component at transcript scale.
* Compare against `../facet-sandbox-poc` (QuickJS) and native.
*/
import ivm from "isolated-vm";
import { readFileSync } from "node:fs";
import { createCompartment } from "./membrane.ts";
// ── membrane crossing ─────────────────────────────────────────────────────
{
const c = await createCompartment();
await c.endow("__h", { echo: (v: unknown) => v });
c.load(`globalThis.spin = function (n) { for (var i = 0; i < n; i++) __h.echo("x"); };`);
c.load(`spin(2000);`);
const N = 20000, t = Date.now();
c.load(`spin(${N});`, 60000);
console.log(`membrane crossing: ${((Date.now() - t) * 1000 / N).toFixed(1)} µs`);
c.dispose();
}
// ── isolate creation ──────────────────────────────────────────────────────
{
const before = process.memoryUsage().rss, t = Date.now();
const held = Array.from({ length: 50 }, () => {
const iso = new ivm.Isolate({ memoryLimit: 32 });
const ctx = iso.createContextSync();
ctx.evalSync(`var c = { render: function () { return ["x"]; } };`);
return iso;
});
console.log(`50 isolates: ${(((process.memoryUsage().rss - before) / 50) / 1024).toFixed(0)} KB each, ${Date.now() - t} ms total`);
for (const i of held) i.dispose();
}
// ── real Markdown at transcript scale ─────────────────────────────────────
{
const iso = new ivm.Isolate({ memoryLimit: 512 });
const ctx = iso.createContextSync();
ctx.global.setSync("global", ctx.global.derefInto());
const t = Date.now();
ctx.evalSync(readFileSync(new URL("./markdown-bundle.js", import.meta.url), "utf8"), { timeout: 60000 });
console.log(`146 KB bundle load: ${Date.now() - t} ms`);
for (const [label, pts] of [["small", 2], ["typical", 8], ["large", 35]] as [string, number][]) {
ctx.evalSync(`__bench(${pts}, 20, 50)`, { timeout: 120000 }); // warm the JIT
const ms = ctx.evalSync(`__bench(${pts}, 300, 3)`, { timeout: 300000 });
console.log(`300 markdown components, ${label.padEnd(8)} ${ms.toFixed(0)} ms per full repaint`);
}
iso.dispose();
}

22
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/demo-facet.ts vendored
View File

@@ -1,22 +0,0 @@
import { readFileSync } from "node:fs";
import { createCompartment } from "./membrane.ts";
const mounted = new Map(), commands = new Map(), notifications = [], subs = [];
const Tui = {
slots: { claim: (s, f) => mounted.set(s, f) },
commands: { add: (n, h) => commands.set(n, h) },
notify: (m) => notifications.push(m),
};
const Transcript = { tail: { subscribe: (f) => subs.push(f) } };
const c = await createCompartment();
await c.endow("__host", { use: (t) => (t === "Tui" ? Tui : Transcript) });
c.load(readFileSync(new URL("./facet-example.js", import.meta.url), "utf8"));
c.load(`globalThis.__r = __facet.construct({ use: function (t) { return __host.use(t); } });`);
console.log("contributions:", { slots: [...mounted.keys()], commands: [...commands.keys()], subs: subs.length });
const comp = mounted.get("footer")({ label: "presses" });
console.log(comp.render(30).join("\n"));
comp.handleInput("abc");
console.log(comp.render(30).join("\n"));
console.log("command ->", commands.get("footer.reset")());
subs[0]({ entries: [1, 2, 3] });
console.log("notifications:", notifications);
c.dispose();

63
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/escape-audit.ts vendored
View File

@@ -1,63 +0,0 @@
/**
* Escape audit.
*
* The claim the membrane makes is that the guest's entire view of the host is
* `{ number, string }` — no object graph, so nothing to walk. This tries to
* break that, including the specific `Reference` footgun that makes isolated-vm
* dangerous when used naively.
*/
import { createCompartment } from "./membrane.ts";
const c = await createCompartment();
const seen: string[] = [];
await c.endow("__h", {
sink: (v: string) => seen.push(v),
give: () => ({ nested: { a: 1 } }),
giveFn: () => () => "host function result",
});
const probe = (name: string, src: string) => {
c.load(`globalThis.__p = String(${src});`);
c.load(`__h.sink(__p);`);
const got = seen.pop();
const bad = got !== undefined && /GOT (?!undefined)|INVOKED|CALLABLE|^object$/.test(got) && !/blocked/.test(got);
console.log(` ${bad ? "!!" : "ok"} ${name.padEnd(34)} ${got}`);
return !bad;
};
console.log("Ambient authority:");
let ok = true;
ok = probe("require / process / fetch", `JSON.stringify([typeof require, typeof process, typeof fetch])`) && ok;
ok = probe("global names visible", `Object.getOwnPropertyNames(globalThis).length + " globals"`) && ok;
console.log("\nClassic escape ladders:");
ok = probe("Function('return process')()", `(function(){ try { return String((function(){}).constructor("return typeof process")()); } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("constructor walk on host data", `(function(){ try { var o = __h.give(); return String(o.constructor.constructor("return typeof process")()); } catch(e){ return "blocked: "+e.name; } })()`) && ok;
console.log("\nThe isolated-vm Reference footgun:");
ok = probe("__invokeRef.deref()", `(function(){ try { return "GOT " + typeof __invokeRef.deref(); } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("__invokeRef.copySync()", `(function(){ try { return "GOT " + typeof __invokeRef.copySync(); } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("__invokeRef.getSync('constructor')", `(function(){ try { return "GOT " + typeof __invokeRef.getSync("constructor"); } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("derefInto() is callable?", `(function(){ var d = __invokeRef.derefInto(); return typeof d === "function" ? "CALLABLE" : "inert (" + typeof d + ")"; })()`) && ok;
ok = probe("invoke derefInto() result", `(function(){ try { __invokeRef.derefInto()(); return "INVOKED"; } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("host globals via derefInto()", `(function(){ var d = __invokeRef.derefInto(); return String(d.process || d.global || "no host globals"); })()`) && ok;
console.log("\nWhat a host FUNCTION looks like to the guest:");
ok = probe("typeof a returned host fn", `(function(){ var f = __h.giveFn(); return typeof f; })()`) && ok;
// NOTE: `f.deref` is NOT undefined — the membrane's proxy returns a proxy for
// every property. That is not a leak: calling it routes to hostFn["deref"],
// which does not exist, and throws host-side. Testing for the property's
// absence gave a false positive in a first draft of this audit.
ok = probe("calling f.deref() reaches nothing", `(function(){ try { var f = __h.giveFn(); f.deref(); return "INVOKED"; } catch(e){ return "blocked: "+e.name; } })()`) && ok;
ok = probe("calling the host fn works", `(function(){ var f = __h.giveFn(); return String(f()); })()`) && ok;
console.log("\nResource bounds:");
const t0 = Date.now();
let interrupted = false;
try { c.load(`while(true){}`, 200); } catch { interrupted = true; }
console.log(` ${interrupted ? "ok" : "!!"} spinning guest interrupted after ${Date.now() - t0}ms`);
console.log(` ok isolate usable afterwards ${c.load(`1+1`) === 2}`);
c.dispose();
console.log(`\n${ok && interrupted ? "AUDIT PASSED" : "AUDIT FAILED"}`);
process.exit(ok && interrupted ? 0 : 1);

42
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/facet-example.js vendored
View File

@@ -1,42 +0,0 @@
// ── REAL facet code. Nothing here knows it is sandboxed. ────────────────────
// No imports rewritten, no string marshalling, no __dispatch. It uses the
// host API exactly as facets.md §11.1 describes it.
class FooterComponent {
constructor(props) { this.label = props.label; this.count = 0; }
render(width) {
const text = ` ${this.label}: ${this.count} `;
const pad = Math.max(0, width - text.length - 2);
const bar = "\u2500".repeat(Math.max(0, width - 2));
return [`\u250c${bar}\u2510`, `\u2502${text}${" ".repeat(pad)}\u2502`, `\u2514${bar}\u2518`];
}
handleInput(data) { this.count += data.length; }
}
const facet = {
id: "@demo/footer:tui",
uses: ["Tui", "Transcript"],
construct(ctx) {
const tui = ctx.use("Tui");
const transcript = ctx.use("Transcript");
// A factory crossing guest → host, by reference.
tui.slots.claim("footer", (props) => new FooterComponent(props));
// A command handler, likewise.
tui.commands.add("footer.reset", () => {
tui.notify("footer reset");
return "ok";
});
// A subscription: host calls back into the guest.
transcript.tail.subscribe((tail) => {
tui.notify(`transcript now has ${tail.entries.length} entries`);
});
return [];
},
};
globalThis.__facet = facet;

34
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/iso-vs-ctx.ts vendored
View File

@@ -1,34 +0,0 @@
import ivm from "isolated-vm";
const rss = () => process.memoryUsage().rss / 1024 / 1024;
// 1. Are two CONTEXTS in one isolate isolated from each other?
{
const iso = new ivm.Isolate({ memoryLimit: 64 });
const a = iso.createContextSync(), b = iso.createContextSync();
a.evalSync(`globalThis.secret = "from-A"`);
console.log("context B sees A's global:", b.evalSync(`typeof globalThis.secret`));
// Can a spin in A be timed out without harming B?
try { a.evalSync(`while(true){}`, { timeout: 100 }); } catch { /* expected */ }
console.log("B usable after A was interrupted:", b.evalSync(`1+1`) === 2);
// Does A's memory bomb kill B?
try { a.evalSync(`var x=[]; for(;;) x.push(new Array(50000).fill(1));`, { timeout: 15000 }); }
catch (e: any) { console.log("A bomb:", e.message.slice(0, 45)); }
try { console.log("B after A's bomb:", b.evalSync(`1+1`)); }
catch (e: any) { console.log("B after A's bomb: DEAD -", e.message.slice(0, 40)); }
try { iso.dispose(); } catch {}
}
// 2. Cost per facet, both ways.
for (const mode of ["one isolate, N contexts", "N isolates"]) {
const before = rss(), t = Date.now();
const held: any[] = [];
const shared = mode.startsWith("one") ? new ivm.Isolate({ memoryLimit: 512 }) : null;
for (let i = 0; i < 50; i++) {
const iso = shared ?? new ivm.Isolate({ memoryLimit: 32 });
const ctx = iso.createContextSync();
ctx.evalSync(`var c = { render: function () { return ["x"]; } };`);
held.push(iso);
}
console.log(`50 facets, ${mode.padEnd(24)}: ${(((rss() - before) / 50) * 1024).toFixed(0)} KB each, ${Date.now() - t} ms`);
if (shared) { try { shared.dispose(); } catch {} } else for (const i of held) { try { i.dispose(); } catch {} }
}

27
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/leak.ts vendored
View File

@@ -1,27 +0,0 @@
import { createCompartment } from "./membrane.ts";
const c = await createCompartment();
let seen = 0;
await c.endow("__l", { drop: (_fn: unknown) => { seen++; } });
c.load(`globalThis.churn = function (n) { for (var i = 0; i < n; i++) __l.drop(function () { return i; }); };`);
const rss = () => process.memoryUsage().rss / 1024 / 1024;
const before = rss();
c.load(`churn(20000);`, 60000);
const peak = c.debugRefCount();
const afterChurn = rss();
globalThis.gc?.(); await new Promise((r) => setTimeout(r, 100));
globalThis.gc?.(); await new Promise((r) => setTimeout(r, 100));
const settled = c.debugRefCount();
console.log(`20000 crossings: host called ${seen}x`);
console.log(` guest refs: ${peak.guestRefs} -> ${settled.guestRefs}`);
console.log(` host table: ${peak.hostTable} -> ${settled.hostTable}`);
console.log(` rss: ${before.toFixed(0)} -> ${afterChurn.toFixed(0)} -> ${rss().toFixed(0)} MB`);
// Cost of the release path: one evalSync crossing per finalized ref.
const t = Date.now();
c.load(`churn(5000);`, 60000);
globalThis.gc?.(); await new Promise((r) => setTimeout(r, 200));
console.log(` 5000 more + gc: ${Date.now() - t} ms, refs now ${c.debugRefCount().guestRefs}`);
c.dispose();

3888
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/markdown-bundle.js vendored
View File

File diff suppressed because it is too large Load Diff

197
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/membrane.ts vendored
View File

@@ -1,197 +0,0 @@
import ivm from "isolated-vm";
/**
* A membrane between the host realm and an isolated-vm isolate.
*
* ── The safety argument, which is structural, not a convention ──────────────
*
* isolated-vm CAN be used unsafely. The documented footgun is handing the guest
* a live handle to a host object — via `derefInto()`, or by returning a
* `Reference` from a host call — after which the guest can walk that object's
* prototype chain into the host realm.
*
* This membrane makes that impossible by construction rather than by care:
*
* 1. `encode()` is the ONLY way a host value reaches the guest, and it is
* `JSON.stringify` with a replacer. Its output is a string. A string
* cannot carry a reference.
* 2. Host callables do not cross. They are replaced by an integer id into
* `hostTable`, which lives entirely host-side. The guest receives a number.
* 3. Exactly ONE `Reference` is ever given to the guest — `__invokeRef`, the
* single call-in point — and its `deref()` throws cross-isolate ("Cannot
* dereference this from current isolate"). Its `derefInto()` yields an
* inert `Dereference` marker: not callable, no host globals.
* 4. `derefInto()` is used exactly once, on the guest's OWN global. No host
* object is ever an argument to it.
*
* The consequence: the guest's entire view of the host is `{ number, string }`.
* There is no object graph to walk, so there is nothing to walk it into. This is
* the same property the QuickJS membrane has, and it is why the choice between
* them is about performance and packaging rather than about safety.
*
* Verified by `src/escape-audit.ts`.
*/
export async function createCompartment({ memoryLimit = 256 } = {}) {
const isolate = new ivm.Isolate({ memoryLimit });
const ctx = await isolate.createContext();
await ctx.global.set("global", ctx.global.derefInto()); // guest's own global only
const hostTable = new Map();
let nextHostId = 1;
const exposeHost = (o) => { const id = nextHostId++; hostTable.set(id, o); return id; };
const guestProxies = new Map();
const finalizer = new FinalizationRegistry((id) => {
guestProxies.delete(id);
if (!released) ctx.evalSync(`delete __refs[${JSON.stringify(String(id))}]`);
});
let released = false;
// Async: `applySync` is synchronous, so a host Promise cannot itself cross.
// Instead the host returns a token, the guest builds a real Promise against
// it, and the host settles that Promise by calling back in when it resolves.
let nextPending = 1;
function encodePending(promise) {
const token = nextPending++;
promise.then(
(v) => settle(token, true, v === undefined ? "" : encode(v)),
(e) => settle(token, false, JSON.stringify(String(e?.message ?? e))),
);
return JSON.stringify({ __pending: token });
}
function settle(token, ok, json) {
if (released) return;
ctx.evalSync(`__settle(${token}, ${ok}, ${JSON.stringify(json)})`);
}
function encode(v) {
if (v && typeof v.then === "function") return encodePending(v);
return JSON.stringify(v, (_k, val) => {
if (typeof val === "function") return { __ref: exposeHost(val) };
if (val && typeof val === "object" && !Array.isArray(val) && hasMethods(val))
return { __obj: exposeHost(val), keys: methodNames(val) };
return val;
});
}
const decode = (json) => JSON.parse(json, (_k, v) => reviveGuest(v));
function intern(id, build) {
const hit = guestProxies.get(id)?.deref();
if (hit !== undefined) return hit;
const p = build();
guestProxies.set(id, new WeakRef(p));
finalizer.register(p, id);
return p;
}
function reviveGuest(v) {
if (!v || typeof v !== "object") return v;
if (typeof v.__guestRef === "number") { const r = v.__guestRef; return intern(r, () => (...a) => callGuestRef(r, a)); }
if (typeof v.__guestObj === "number") {
const r = v.__guestObj;
return intern(r, () => { const o = {}; for (const k of v.keys) o[k] = (...a) => callGuestRef(r, [k, a]); return o; });
}
return v;
}
function callGuestRef(ref, args, timeout = 5000) {
const out = ctx.evalSync(
`__refs[${JSON.stringify(String(ref))}](${JSON.stringify(JSON.stringify(args))})`,
{ timeout },
);
return out === "" || out === undefined ? undefined : decode(out);
}
// The single host entry point. A Reference, applied synchronously with copied
// arguments — no host object graph is ever exposed.
const invoke = new ivm.Reference((idJson, pathJson, argsJson) => {
const path = JSON.parse(pathJson);
let target = hostTable.get(JSON.parse(idJson));
for (let i = 0; i < path.length - 1; i++) target = target[path[i]];
const result = path.length ? target[path[path.length - 1]](...decode(argsJson)) : target(...decode(argsJson));
return result === undefined ? "" : encode(result);
});
await ctx.global.set("__invokeRef", invoke);
await ctx.eval(`
globalThis.__refs = {}; globalThis.__nextRef = 1;
globalThis.__outgoing = new WeakMap();
globalThis.__methodNames = function (o) {
var out = [], p = o;
while (p && p !== Object.prototype) {
for (const k of Object.getOwnPropertyNames(p))
if (k !== "constructor" && typeof o[k] === "function" && out.indexOf(k) < 0) out.push(k);
p = Object.getPrototypeOf(p);
}
return out;
};
globalThis.__mkRef = function (fn) {
var prior = __outgoing.get(fn); if (prior !== undefined) return { __guestRef: prior };
var id = __nextRef++; __outgoing.set(fn, id);
__refs[String(id)] = function (j) { var r = fn.apply(null, JSON.parse(j)); return r === undefined ? "" : __encode(r); };
return { __guestRef: id };
};
globalThis.__mkObjRef = function (obj) {
var prior = __outgoing.get(obj);
if (prior !== undefined) return { __guestObj: prior, keys: __methodNames(obj) };
var id = __nextRef++; __outgoing.set(obj, id);
__refs[String(id)] = function (j) { var c = JSON.parse(j); var r = obj[c[0]].apply(obj, c[1]); return r === undefined ? "" : __encode(r); };
return { __guestObj: id, keys: __methodNames(obj) };
};
globalThis.__encode = function (v) {
return JSON.stringify(v, function (k, val) {
if (typeof val === "function") return __mkRef(val);
if (val && typeof val === "object" && !Array.isArray(val) && __methodNames(val).length) return __mkObjRef(val);
return val;
});
};
globalThis.__pending = {};
globalThis.__settle = function (token, ok, json) {
var d = __pending[token];
if (!d) return;
delete __pending[token];
var value = json === "" ? undefined : JSON.parse(json);
ok ? d.resolve(__revive(value)) : d.reject(new Error(value));
};
globalThis.__revive = function (v) {
if (v && typeof v === "object" && typeof v.__pending === "number") {
var token = v.__pending;
return new Promise(function (res, rej) { __pending[token] = { resolve: res, reject: rej }; });
}
if (v && typeof v === "object") {
if (typeof v.__ref === "number") return __wrap(v.__ref, []);
if (typeof v.__obj === "number") return __wrap(v.__obj, []);
for (var k in v) v[k] = __revive(v[k]);
}
return v;
};
globalThis.__wrap = function (id, path) {
return new Proxy(function () {}, {
get: function (t, prop) {
if (typeof prop !== "string") return undefined;
if (prop === "then" || prop === "toJSON" || prop === "inspect") return undefined;
if (prop === "length" || prop === "name") return t[prop];
return __wrap(id, path.concat([prop]));
},
apply: function (_t, _this, args) {
var out = __invokeRef.applySync(undefined, [JSON.stringify(id), JSON.stringify(path), __encode(args)]);
return out === "" ? undefined : __revive(JSON.parse(out));
},
});
};
`);
return {
async endow(name, obj) { await ctx.eval(`globalThis[${JSON.stringify(name)}] = __wrap(${exposeHost(obj)}, []);`); },
load: (src, timeout = 30000) => ctx.evalSync(src, { timeout }),
debugRefCount: () => ({ hostTable: hostTable.size, guestRefs: ctx.evalSync(`Object.keys(__refs).length`) }),
dispose() {
released = true;
// An isolate that hit its memory limit is already disposed, and calling
// dispose() again throws "Isolate is already disposed".
try { isolate.dispose(); } catch { /* already gone */ }
},
};
}
const RESERVED = new Set(["constructor"]);
const methodNames = (o) => { const out = []; for (let p = o; p && p !== Object.prototype; p = Object.getPrototypeOf(p)) for (const k of Object.getOwnPropertyNames(p)) if (!RESERVED.has(k) && typeof o[k] === "function" && !out.includes(k)) out.push(k); return out; };
const hasMethods = (o) => methodNames(o).length > 0;

137
third_party/pi/packages/agent/docs/mobile-handoff/02-plugins/02-sandbox/src/property-test.ts vendored
View File

@@ -1,137 +0,0 @@
/**
* Property tests for the membrane.
*
* Every bug found by hand while building it failed SILENTLY — wrong output, no
* throw. That is the worst possible failure mode for a boundary, and the reason
* these exist. Run with: node --experimental-strip-types --expose-gc property-test.ts
*/
import { createCompartment } from "./membrane.ts";
let pass = 0;
const failures: string[] = [];
const check = (name: string, ok: boolean, detail = "") => {
if (ok) pass++;
else failures.push(`${name}${detail ? ` — ${detail}` : ""}`);
};
/** Random JsonValue. The membrane claims to round-trip exactly these. */
function randomJson(depth = 0): unknown {
const r = Math.random();
if (depth > 3 || r < 0.25) return Math.floor(Math.random() * 1000) - 500;
if (r < 0.35) return Math.random() * 1e6 - 5e5;
if (r < 0.45) return Math.random() < 0.5;
if (r < 0.5) return null;
if (r < 0.65) {
const alphabet = ['a', 'z', '0', ' ', '"', '\\', '\n', '\u00e9', '\u4e2d', '\u{1f600}'];
let s = "";
const n = Math.floor(Math.random() * 12);
for (let i = 0; i < n; i++) s += alphabet[Math.floor(Math.random() * alphabet.length)];
return s;
}
if (r < 0.8) return Array.from({ length: Math.floor(Math.random() * 5) }, () => randomJson(depth + 1));
const o: Record<string, unknown> = {};
for (const k of ["a", "b", "c", "d", "__proto__x", "0", "length"]) {
if (Math.random() < 0.5) o[k] = randomJson(depth + 1);
}
return o;
}
const c = await createCompartment();
// ── 1. Value round-trip, both directions ──────────────────────────────────
const echoed: unknown[] = [];
await c.endow("__h", { echo: (v: unknown) => v, sink: (v: unknown) => { echoed.push(v); } });
c.load(`globalThis.roundTrip = function (v) { return __h.echo(v); };
globalThis.sendBack = function (v) { __h.sink(v); };`);
for (let i = 0; i < 400; i++) {
const value = randomJson();
const json = JSON.stringify(value);
if (json === undefined) continue;
// host -> guest -> host
c.load(`globalThis.__t = roundTrip(${json});`);
c.load(`sendBack(__t);`);
const got = echoed.pop();
check("value round-trip", JSON.stringify(got) === json, `${json} became ${JSON.stringify(got)}`);
}
// ── 2. Identity: the same guest object must map to the same host proxy ────
const seen: unknown[] = [];
await c.endow("__id", { take: (o: unknown) => { seen.push(o); } });
c.load(`globalThis.shared = { m: function () { return 1; } };
globalThis.sendShared = function () { __id.take(shared); __id.take(shared); };`);
c.load(`sendShared();`);
check("identity preserved across crossings", seen[0] === seen[1]);
c.load(`sendShared();`);
check("identity stable across separate calls", seen[0] === seen[2]);
// ── 3. Callables survive and remain callable ──────────────────────────────
let held: any;
await c.endow("__f", { keep: (fn: unknown) => { held = fn; } });
c.load(`__f.keep(function (a, b) { return a + b; });`);
check("guest function callable from host", held(2, 3) === 5, `got ${held?.(2, 3)}`);
// ── 4. Errors cross with name and message ─────────────────────────────────
c.load(`__f.keep(function () { throw new TypeError("boom"); });`);
let msg = "";
try { held(); } catch (e: any) { msg = e.message; }
check("guest throw reaches host", msg.includes("boom"), `got "${msg}"`);
// ── 5. Reference release: crossings must not leak ─────────────────────────
await c.endow("__l", { drop: (_fn: unknown) => {} });
c.load(`globalThis.churn = function () { for (var i = 0; i < 500; i++) __l.drop(function () { return i; }); };`);
c.load(`churn();`);
const beforeGc = c.debugRefCount().guestRefs;
globalThis.gc?.();
await new Promise((r) => setTimeout(r, 60));
globalThis.gc?.();
await new Promise((r) => setTimeout(r, 60));
const afterGc = c.debugRefCount().guestRefs;
check("dropped references are released", afterGc < beforeGc / 2, `${beforeGc} -> ${afterGc}`);
// ── 5b. Prototype pollution must not cross ────────────────────────────────
// JSON.parse is safe (it makes __proto__ an own property), but a membrane that
// assigns parsed keys onto a fresh object is not.
const before = ({} as any).polluted;
c.load(`__h.sink(JSON.parse('{"__proto__":{"polluted":true}}'));`);
check("guest cannot pollute host Object.prototype", ({} as any).polluted === before);
await c.endow("__p", { give: () => JSON.parse('{"__proto__":{"hostPolluted":true}}') });
await c.endow("__q", { show: (v: string) => check("host cannot pollute guest prototype", v === "undefined", v) });
c.load(`var got = __p.give(); __q.show(String(({}).hostPolluted));`);
// ── 6. Interrupt budget, and the VM survives it ───────────────────────────
// Direct: the budget on this call bounds it.
{
const t0 = Date.now();
try { c.load(`while (true) {}`, 100); } catch { /* expected */ }
check("runaway guest code is interrupted", Date.now() - t0 < 600, `took ${Date.now() - t0}ms`);
}
// Re-entrant: a guest function invoked BY THE HOST gets `callGuestRef`'s own
// budget, not the outer one. Budgets do not nest — see README "Limitations".
c.load(`globalThis.spin = function () { while (true) {} };`);
await c.endow("__s", { run: (fn: any) => { try { fn(); return "returned"; } catch { return "threw"; } } });
{
const t0 = Date.now();
try { c.load(`globalThis.__x = __s.run(spin);`, 50); } catch { /* expected */ }
const took = Date.now() - t0;
check("re-entrant call uses its OWN budget, not the outer one", took > 1000,
`took ${took}ms — if this is now <1s the nesting behaviour changed`);
}
c.load(`globalThis.__alive = 1 + 1;`);
check("VM usable after an interrupt", true);
// ── 7. Async, and clean teardown with a pending promise ───────────────────
await c.endow("__a", { never: () => new Promise(() => {}), soon: () => Promise.resolve("v") });
c.load(`globalThis.out = null; __a.soon().then(function (v) { out = v; });`);
await new Promise((r) => setTimeout(r, 30));
await c.endow("__o", { show: (v: string) => check("async host result reaches guest", v === '"v"', v) });
c.load(`__o.show(JSON.stringify(out));`);
c.load(`__a.never().then(function () {}, function () {});`); // deliberately unsettled
let disposeOk = true;
try { c.dispose(); } catch { disposeOk = false; }
check("dispose with a pending promise does not crash", disposeOk);
console.log(`\n${pass} passed, ${failures.length} failed`);
for (const f of failures.slice(0, 10)) console.log(" FAIL", f);
process.exit(failures.length === 0 ? 0 : 1);

81
third_party/pi/packages/agent/docs/mobile-handoff/README.md vendored
View File

@@ -1,81 +0,0 @@
# pi — design handoff
Work in numbered order. Each unit is self-contained and independently testable;
later units consume earlier ones.
```
01-harness/
01-delta/ op vocabulary, tracker, applier, codec [LANDED IN CHORD]
02-scopes/ storage scopes and list tags [STEP 1 ACTIONABLE]
03-execenv/ bounded Shell output, capture, spill [PRODUCTION CODE + TESTS]
04-tool-output/ the ToolOutput sink [SPEC ONLY]
05-assistant-output/ assistant partials, symmetric with 04 [SPEC ONLY]
02-plugins/
01-facets/ the facet system [SPEC ONLY]
02-sandbox/ isolated-vm membrane [CODE + 412 tests]
```
Base everything on a clean checkout of `origin/dev`.
## Read this first
**Three units ship working code. Four are specifications.** The table above says
which. Do not assume a doc describes something that exists.
**`01-delta/FINDINGS.md` is historical evidence, not an implementation queue.** Production lives in `packages/chord/src/delta/index.ts`: flush-time dirty tracking fixed D1, and production remeasurement closed D2. The explicit append/truncate API is rejected; see [`01-harness/01-delta/append-decision.md`](01-harness/01-delta/append-decision.md). The code beside the handoff remains prototype and benchmark evidence.
**If a doc and the code disagree, the code wins** — fix the doc and say so in the
commit.
**Port the tests before the implementation.** Each group's comment explains the
failure it guards against, and several of those failures are silent: wrong output,
no exception.
**Benchmark with `node --experimental-strip-types`, never through a transpiler.**
Measuring this module through `tsx` inflates results 2.6x. `FINDINGS.md` D5 lists
five more measurement traps, each of which produced a confident wrong conclusion.
## Unit status
| unit | ships | state |
| --- | --- | --- |
| **01-delta** | production implementation and tests in `packages/chord`; prototype evidence here | landed; D1 fixed, explicit text API rejected after production remeasurement |
| **02-scopes** | spec + [actionable Step 1 handoff](01-harness/02-scopes/implementation-handoff.md) + `scopes.variance.ts` | Step 1 scopes/list tags actionable, not implemented; JSONL Chord encoding/address interning deferred to separately approved Step 2 |
| **03-execenv** | production implementation in `packages/agent`; prototype evidence here | source-bounded adaptive output, lazy spill backpressure, and bash migration landed; bash's temporary checkpoint cadence moves to `ToolOutput` next |
| **04-tool-output** | spec + design notes | **not built.** The piece every measurement of the op encoding depends on |
| **05-assistant-output** | spec | not built. Same shape as 04; do it after |
| **02-plugins/01-facets** | spec, ~1800 lines | not built. §14 rewritten to match the sandbox PoC |
| **02-plugins/02-sandbox** | working PoC, 412 assertions | `npm install && npm run audit` |
## Suggested order
1. **`02-scopes` Step 1** — follow the [actionable implementation handoff](01-harness/02-scopes/implementation-handoff.md); stop for approval before its separate Step 2.
2. **`04-tool-output`** — reuse the landed adaptive publisher for generic tools, Chord event/durable batches, terminal flushes, and atomic memo checkpoints.
3. **05**, then **02-plugins**.
## Live bugs on `origin/dev`, independent of this design
- `drive/tools.ts:257` — `clearReplayCheckpoint` deletes `pendingToolOutput` before
re-executing a replay-safe tool. Memos exist so a replayed tool skips work, and
skipped work emits nothing, so output for memoised work is lost today. Seed from
it instead (`harness-tools.md` §7.4).
- `runtime/progress.ts:44` — `commitWrite(item)` captures the value at call time
and writes fire-and-forget, so an older checkpoint can land after a newer one.
- memo and checkpoint are two transactions (`drive/tools.ts:112` vs
`progress.ts:44`). They must be one (`harness-tools.md` §7.5).
**Expected test churn:** nine tests assert the old cleanup write set and will fail
once `retireScope` replaces the per-address deletes. That is the change landing.
## Environment
- Node 22+ for every shipped `.ts` file. They run under
`node --experimental-strip-types` with no build step and no dependencies.
- In the pi repo, tests run from the package:
`cd packages/agent && npx vitest run --config vitest.harness.config.ts`.
The root vitest config does **not** alias `@earendil-works/pi-ai`; the
per-package harness config does.
- Typecheck with `npx tsgo --noEmit` from the repo root. **Baseline is ~788
pre-existing errors**, almost all in `packages/ai/test`. Count only:
`grep "error TS" | grep -E "packages/(agent|session-backends)/src"`.
- `packages/ai` cannot be built offline — model data is fetched at build time.

1158
third_party/pi/packages/agent/docs/plugins.md vendored
View File

File diff suppressed because it is too large Load Diff

254
third_party/pi/packages/agent/docs/post-wp05-roadmap.md vendored
View File

@@ -1,254 +0,0 @@
# Post-WP05 roadmap audit
**Audit baseline:** `5507d76ee` (`dev`, 2026-08-27).
**Status:** Planning inventory, not a behavior contract. [`harness.md`](harness.md) remains normative where it agrees with the current product boundary. Contradictions listed below must be resolved explicitly; this document does not silently choose one side.
## Scope and method
This audit covers the durable AgentHarness and its directly coupled Session backends and presentation path:
- `packages/agent/src/harness`, `packages/agent/src/search`, and their tests/docs;
- `packages/session-backends/sqlite-node`;
- the current `packages/protocol`, `packages/client`, and `packages/server` presentation path;
- `packages/coding-agent/src/experimental` and its focused tests where they host that path;
- telemetry plumbing in `packages/telemetry`, `packages/ai`, and `packages/agent`.
The inventory was checked against current source, tests, package READMEs, the complete `harness.md`, completed WP00–WP07 status, the actionable WP08 handoff, explicit stubs/TODOs/skips, and the shipped package boundaries. Historical handoffs are not treated as backlog when current source and the completed WP05 contract supersede them.
## Executive result
WP05 is complete through M10. Its remaining assistant-output work is owned by the [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md) and its numbered prerequisites. The current Harness execution graph has no unfinished runtime path: `watchSession()` is the sole `SliceNotImplemented` Harness method.
That does **not** mean the surrounding durable system is complete. The remaining audit findings are:
1. normative JSONL snapshot-compaction behavior with no implementation;
2. one required Harness method stub (`watchSession`);
3. a deliberate removal of raw `RemoteSession` that conflicts with later normative WP06/`harness.md` text;
4. a public search type skeleton that conflicts with the newer search design and has no implementation;
5. a full telemetry vocabulary whose only production span is the tool-hook span;
6. smaller repository, client-watch, query-bound, documentation, and end-to-end-test gaps.
WP07 completed SQLite host-ownership alignment and live-source fork support after the audit baseline; its historical handoff is [`work-packages/07-sqlite-host-ownership-live-forks.md`](work-packages/07-sqlite-host-ownership-live-forks.md). WP08 now owns the separate named-branch, tree-state, and bounded-memory fork redesign; its actionable handoff is [`work-packages/08-named-branch-streaming-forks.md`](work-packages/08-named-branch-streaming-forks.md).
## Required missing functionality and contract contradictions
### R12 — Session-wide Harness watch
**Evidence**
- `AgentHarness.watchSession(context)` is public in `src/harness/agent-harness.ts`.
- `Harness.watchSession()` throws `SliceNotImplemented("watchSession")` in `src/harness/runtime/harness.ts`.
- `SessionSnapshot` currently contains only `{ lanes: LaneInfo[]; faulted: boolean }`.
- Lane watch, event buffering, delivery-tail barriers, and `resnapshot()` already exist in `src/harness/events.ts` and `src/harness/runtime/lane.ts`.
**Remaining boundary**
Define one coherent capture and fold for dynamic lane inventory and fault state. Decide whether the intentionally small `SessionSnapshot` stays small or gains session metadata/stats/global configuration. Then implement snapshot-before-events, lane creation, resnapshot, listener reentrancy, close/fault behavior, and a session reducer if event-only replication is promised.
**Dependency**
Independent of the mobile assistant-output handoff and SQLite internals. It should precede any revisioned Transcript service or remote session-wide observation built on it.
### JSONL snapshot compaction
**Evidence**
`harness.md` §1.7 normatively specifies temp-file-and-rename snapshot compaction, preserved sequence high-water marks/list cursors, threshold checks on open, and reclamation after terminal/outcome deletions. `JsonlStorage` implements atomic creation, torn-tail repair, legacy-v3 rewrite, append, and fork snapshots, but no current-state snapshot rewrite or dead-byte accounting exists.
**Consequence**
Superseded `pi.op.state`, deleted pending payloads, deleted tool checkpoints, and deleted assistant-frame lists remain physical bytes indefinitely. Generic compaction and the [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md) are complementary. Compaction reclaims dead session-scoped write history after the fact; the handoff moves pending assistant/tool output into ephemeral scopes so it never becomes main-log history, and replaces full/per-frame replication with Chord op batches.
**Dependency**
The assistant-output handoff does not depend on J1: scoped storage is the intended lifetime mechanism for pending output, while J1 remains the reclamation mechanism for superseded session-scoped state. Measure both independently so their effects are not conflated.
### Remote Session contract contradiction — decision required
**Current product boundary**
Commit `f8a6e670d` deliberately deleted `RemoteSession`, its raw Session RPC protocol, and the server mutation-scope manager, replacing them with attachment-fenced routed semantic services. Current protocol/server READMEs explicitly state that real `Session` and `AgentHarness` objects remain process-local. The shipped path supports Session discovery/creation/attachment, main-lane prompt/watch, and allowlisted plugin-service calls; it does not expose `Session`, `SessionMutation`, values/lists, branches, or storage over RPC.
**Conflicting contract**
WP06, written after that deletion, requires the “current keyless RemoteSession mutation transport,” includes remote begin/read/commit/publication/end in required tests and stop conditions, and forbids its removal. `harness.md` §§2.8 and 9.1 likewise state that local and remote implementations preserve that lifecycle. No such implementation, protocol schema, client facade, server-held scope, worker adapter, or conformance test exists at the audit baseline.
**Required decision**
Choose one before scheduling implementation:
1. **Process-local Session remains intentional:** remove the false RemoteSession requirements from normative current-state docs while preserving semantic service RPC; or
2. **RemoteSession is required:** commission a dedicated package for mutation begin/read/commit/end, disconnect/timeout cleanup, publication-before-end, values/lists/branches/entries/stats, protocol validation, client/server/worker adapters, and remote conformance.
Do not count the current lane-watch compatibility RPC or plugin-service RPC as RemoteSession. Do not restore the deleted 507-line facade unchanged: it predates the keyless Session/Branch contract and relied heavily on untyped decoding.
### Telemetry contract exceeds implementation
**Evidence**
`src/harness/telemetry.ts` and generated `docs/telemetry-schema.md` declare `pi.ai.request`, operation, checkpoint, turn, step, tool, hook, sleep, event-handler, and session-write spans. Production source starts only `pi.harness.hook`, and only for registered `before_tool`/`after_tool` handlers. AI options propagate `telemetryContext`, but no provider path starts `pi.ai.request`. Server request ingress has cancellation but no trace carrier or client/server RPC spans. `TODO_CONTEXT` remains at transport/worker lifecycle and event-delivery boundaries.
**Remaining boundary**
Treat this as separate packages:
1. local Harness/Session/AI instrumentation and runtime tests;
2. RPC trace-carrier/client/server propagation;
3. an optional application-selected exporter/adapter.
First reconcile whether every declared span is still wanted. If retained, implement it; if not, remove the unsupported public schema surface and correct `harness.md`. Do not mix telemetry with Context/RPC cancellation, which already has independent request-ID signaling.
### S3 — Search
**Evidence**
`src/search/index.ts` exports a public `SessionSearchService` skeleton with `sync()`, `notify()`, and array-returning `searchEntries()`. `harness.md` §2.8 instead specifies a standalone service, separate catch-up/notify utilities, generation-aware cursors, and optional `AsyncIterable` entry search. There is no factory, sync utility, cursor store, projection, or source SQLite FTS implementation. At the audit baseline the SQLite README advertised nonexistent `createSqliteSessionSearch()` behavior; this audit corrected that README rather than treating the absent API as implemented.
**Remaining boundary**
Before implementation, replace or reconcile the draft public interface and decide metadata filtering (`cwd`), candidate restriction, or indexed metadata. Post-filtering after ranked `limit` is unsound. Then implement standalone catch-up and a separate SQLite FTS5 projection; do not add repository search methods.
### R11 — Schema migrations, activation-gated
No current format-4 migration is required. Memory is current-only; JSONL and SQLite reject unsupported storage versions; SQLite runs only idempotent `001_initial.sql`. R11 becomes required immediately before the first incompatible durable storage version/address/state change after format 4 is stabilized. It is not prerequisite work for the mobile assistant-output handoff or current WIP format replacement.
When activated, it must provide ordered transactional migrate-on-open under exclusive ownership, version-specific JSONL decoding plus post-migration compaction, and total mappings for every reachable open operation leaf and surviving value/list.
## Correctness and data-safety debt
### SQLite host ownership and live-source forks — completed by WP07
The authoritative product rule is in `plugins.md`: exactly one host-assigned process owns writable Session authority; normally it is the Session worker, while the server may temporarily own a newly created or forked destination before closing it and handing it off. Storage backends do not implement writer ownership. The server closes a worker before destructive repository administration.
SQLite now follows that rule: the writer-lease schema/module, claims, renewal timer, lease-loss path, and pre-commit callback are gone, with no replacement lock or ownership primitive. Create/open/fork/delete retain repository-local ID reservation. Metadata open and deletion use a true no-create read-write mode; listing and external fork sources use no-create read-only connections.
Same-repository forks retain the source `commitQueue` ordering seam. A source owned elsewhere, including a live worker, is read from its exact canonical container through one independent read-only deferred WAL transaction. Focused per-file and shared-container tests commit a complete later source transaction after the reader establishes its snapshot and before it closes: the first fork excludes the transaction wholly and a later fork includes it wholly.
WP07 also completed canonical `(containerPath, sessionId)` active identity, safe explicit-ID filenames, custom `databasePath` parent creation, Session-scoped shared deletion, WAL/SHM cleanup, and all-settled SQLite repository close. Writable open/delete reject foreign metadata; foreign fork sources are read only from their exact path. Equal-`createdAt` list ordering still has no deterministic tie-break and remains a later behavior-preserving cleanup.
### Repository close ownership is unspecified
`JsonlSessionRepo.close()` contains the only active Agent source TODO and closes no open Session handles. Memory still uses fail-fast `Promise.all`; SQLite now performs backend-local all-settled cleanup of currently open handles, but an already-admitted create/open/fork can still register a handle after repository close captured that set. `SessionRepo` itself declares no `close()` method and shared conformance does not define repository-to-handle ownership or draining of admitted repository operations. Resolve ownership and common cleanup in one repository-lifecycle package; do not patch one backend further without deciding the common contract.
### Client watch staleness after disconnect
`Client` clears its active watch-listener map on disconnect, but existing `LaneWatch` objects keep local `ready`/`started` state and old watch IDs. After reconnect/reattach they can call `start()` or `resnapshot()` and fail remotely instead of deterministically rejecting as stale, contrary to `packages/client/README.md`. Service-subscription objects have the sibling problem: their listeners are cleared and surviving objects become silently dead. Fix both with connection/attachment incarnation fencing and focused reconnect tests. This is independent of R12: the current client method is a compatibility main-lane watch.
### Query bounds and SQLite bind limits
- SQLite `getEntries(ids)` emits one placeholder per requested ID and can exceed the engine’s variable limit.
- Entry, usage, and branch limits use ad hoc `Math.max(0, limit)` behavior. Memory and SQLite diverge for `NaN`, infinities, fractions, and extreme values; unlike list reads, there is no shared normalization contract.
Define cross-backend query-limit semantics in agent conformance, then chunk SQLite ID lookups. This is a storage-contract hardening package, not part of WP07.
### Harness contract and conformance closure
- The public `OperationStatus` includes `"running"`, but lane inspection, snapshots, and `reduceLaneSnapshot` currently produce only `"open"` or `"aborting"`. Define and implement its producer or remove the dead variant.
- The pre-rewrite abort contract bound/published `operation_abort` before resolving the cancellation promise and signalling the live gate. Current `Lane.command()` materializes the result — resolving/signalling — before constructing and binding the event batch, although it still binds recipients before releasing the Session mutation line. Decide whether to change the implementation or retain/document the current no-interleaving order; add an explicit ordering test.
- The production gate-close contract permits only `HarnessClosed | HarnessFault`, but the private source primitive accepts any `Error` and isolated tests use that wider type. Narrow the source declaration and fixtures or explicitly retain the private widening.
- Part 9 of `harness.md` is the required conformance matrix. Existing focused tests cover the graph extensively, including cancellation reconciliation over all 13 leaves, but there is no audited one-to-one proof that every close/reopen leaf case and every race row has both deterministic orders. Audit the matrix and add only the missing cases rather than claiming blanket completion.
Keep this package separate from telemetry, RemoteSession, and the mobile assistant-output handoff; it is local contract/test closure.
### Disabled real worker persistence regression
`packages/coding-agent/test/experimental-remote-runtime.test.ts` still skips “completes and persists a prompt through the worker-owned Harness” with the obsolete note “Re-enable with runtime no-tool execution.” No-tool execution now exists. Re-enable or replace it with a deterministic faux-provider real-worker persistence test; do not use a real paid provider.
## Performance debt
### Mobile assistant-output handoff — durable and replication amplification
The user-supplied motivating mini Session outside the repository is 303,920 bytes across 569 physical lines. These external measurements are evidence, not a reproducible checked-in fixture:
- 477 assistant-frame appends totaling about 118,418 serialized write bytes;
- 12 frame-list deletes; physical lines mentioning the frame namespace total 148,214 bytes;
- about 51,568 bytes of superseded `pi.op.state` writes and 26,192 bytes of one structural preparation, showing why generic JSONL compaction and frame-specific bounding are distinct.
The authoritative design is the [mobile assistant-output handoff](mobile-handoff/01-harness/05-assistant-output/message-update.md), following the numbered `01-harness` prerequisites in [`mobile-handoff/README.md`](mobile-handoff/README.md). Chord delta tracking has landed; scoped storage, tool-output integration, and assistant-output integration have not.
Implementation must add a deterministic repository fixture and measurement script that reproduces or replaces these figures, then measure Memory logical elements, SQLite rows/pages/WAL, JSONL peak sidecar/main-log bytes, and reopen/replay time. The handoff must preserve unknown-outcome recovery, invocation fencing, non-blocking provider streaming, and settlement retirement while eliminating per-frame durable writes and quadratic `message_update` replication. Numerical budgets belong in its implementation handoff/tests, not in a competing standalone design.
### SQLite branch divergence
`createDivergentBranchForEntry()` copies every row after the newest compaction; with no compaction it copies root-through-parent. A first divergence from a long uncompacted transcript is therefore O(history) writes. This exposes an internal contradiction in `harness.md` §2.6: its opening bounded-prefix promise conflicts with its own compaction-based copy algorithm, which the implementation follows. Change both the specification and segment representation so a divergence can reference a covering segment at the parent boundary. Preserve shared-container support and add large uncompacted divergence plus chain-soundness tests/benchmarks.
### Fork contract and materialization — WP08 actionable
All current backends materialize source-sized fork snapshots. Branch scope defaults to `main` without named-Branch ancestry validation; tree scope omits application values/lists; JSONL closed-source fork replay may repair its source. WP08 replaces that contract with required named-branch or tree scope, one closed built-in-state policy, complete current application state for tree forks, and backend-specific bounded-memory copy procedures. It preserves WP07's host ownership, physical identity, same-repository ordering, and independent live-source WAL boundary.
### SQLite catalog, statements, stats, and reclamation
- Default per-session `list()` synchronously opens/configures every SQLite file serially and silently skips failures. A shared container or external catalog is the scalable deployment choice; bounded async scheduling does not make `DatabaseSync` nonblocking.
- Most hot queries prepare a fresh statement each call. Cache narrowly owned statements after measuring.
- Each usage row parses and rewrites the full aggregate JSON usage payload.
- Shared-container row deletion does not reclaim pages. Define maintenance/VACUUM policy separately; never add `VACUUM` to ordinary deletion casually.
These are measurable optimization/operations packages, not correctness fixes.
### Pending-payload amplification and mutation-line parallelism
Queued payloads deliberately write `pendingEntry → immutable entry`; measure pathological payloads before changing it. Keyed Session mutation lines remain optional and require profiling plus a fresh mutable-ownership audit. Neither is current correctness work.
## Behavior-preserving cleanup and documentation repair
Completed by this audit:
- rewrote the SQLite README’s nonexistent `SqliteSessionRepository`/search APIs, wrong `await using` and `appendMessage` example, FTS trigger/rebuild claims, and “one shared connection” claim;
- reconciled `harness.md`’s default one-file SQLite wording with the supported optional shared container;
- reconciled `harness.md` Part 8 with this audit and corrected stale drive-ownership/RPC-cancellation status in `telemetry.md`;
- corrected coding-agent settings docs: install telemetry configuration also controls selected provider attribution headers.
Remaining cleanup:
- **Harness-owned DTO boundary:** the durable Harness still imports legacy agent-loop DTOs from `src/types.ts`: `AgentMessage`, `AgentToolResult`, `AgentToolCall`, `AgentTool`, `QueueMode`, and `ThinkingLevel`. Make a later one-shot cut to independent `HarnessMessage`, `CustomHarnessMessages`, `HarnessToolResult`, `HarnessToolCall`, `HarnessQueueMode`, and `HarnessThinkingLevel` definitions; keep `AgentHarnessTool` as the executable Harness configuration type. Update Harness internals, root exports, declaration-merging tests, documentation, and experimental coding-agent consumers together. Do not alias the new message/tool DTOs back to legacy types, because that preserves the coupling this work is intended to remove.
- Preserve shared-container support; do not remove it incidentally.
- Remove or use unused `insertEntryRow()` and `insertUsageLedgerRow()`.
- Consolidate duplicated SQLite branch payload/structure scan plumbing only after correctness tests pin both paths.
- Decide whether unused `sessions.metadata` and unmeasured indexes have a future owner before schema removal; do not change schema casually.
- Consolidate `startAiSpan()`/`startHarnessSpan()` implementation only if the telemetry surface is retained.
- Mark historical durability documents as delivered where they still read as implementation queues. WP00–WP07, `runtime-simplification.md`, `values.md`’s old consumer deferrals, and external-finalization designs are not active runtime backlog.
## Optional or deferred product capabilities
These are not blockers for the durable Harness:
- standalone S3 search after its API decisions;
- Accounts removal and revisioned Transcript production;
- authenticated workspace/client authorization for the experimental local server;
- private returned references, service flow control, multi-pane presentation, plugin kernel/reload completion, and version-skew negotiation;
- administrative precise rewrite tooling;
- a partitioned Postgres backend/retention policy;
- generic remote Harness/object capabilities;
- DeltaState or delta replication before concrete snapshot pressure justifies it;
- a production telemetry exporter;
- schema migrations before an incompatible stabilized-format change activates R11.
## Recommended dependency order
The order is by data safety first, then dependencies. Independent tracks may proceed in parallel only when they do not edit the same contracts.
1. **Harness contract/conformance closure.** Resolve `OperationStatus.running`, abort signal/event binding order, gate-close typing, and the Part 9 coverage matrix.
2. **Remote Session decision (decision only).** Resolve the false normative boundary early. If process-local wins, repair the docs. If raw RemoteSession wins, later create a dedicated protocol/client/server/worker package; do not fold it into telemetry or R12.
3. **Client watch/subscription staleness** and **repository lifecycle contract.** Small independent correctness packages; complete them before expanding server/worker lifecycle semantics. The lifecycle package must also address Memory's fail-fast repository close.
4. **[Mobile Harness handoff](mobile-handoff/README.md).** Follow its numbered prerequisites through scoped storage, tool output, and assistant output; preserve all recovery boundaries and land deterministic amplification measurements.
5. **JSONL snapshot compaction.** Implement the already-normative physical reclamation path and metrics for remaining session-scoped history.
6. **R12 session-wide watch.** Complete the only Harness method stub before building revisioned Transcript/session-wide remote observation.
7. **Telemetry, if retained:** reconcile schemas, then local instrumentation, then RPC propagation, then an optional exporter. RPC propagation follows the Remote Session/product-boundary decision.
8. **WP08 — named-branch and streaming forks.** Implement the actionable handoff without reopening WP07 ownership or lifecycle decisions.
9. **SQLite branch/query performance hardening.** Keep separate from completed WP07 ownership alignment and WP08 fork semantics; require benchmarks.
10. **S3 search.** Resolve its API/filter/cursor decisions, then implement catch-up and the standalone FTS projection.
11. **R11 migrations.** Activate immediately before the first incompatible stabilized durable schema change, not earlier.
## Stop conditions for roadmap accuracy
This inventory must be updated when any of these facts changes:
- `watchSession` stops being the sole Harness `SliceNotImplemented` method;
- raw RemoteSession is either recommissioned or removed from the normative contract;
- JSONL snapshot compaction lands;
- telemetry schemas are implemented or removed;
- S3’s public API is reconciled;
- a durable format change activates R11;
- WP08 lands or its fork contract changes;
- the host-authority contract changes.

215
third_party/pi/packages/agent/docs/rpc.md vendored
View File

@@ -1,215 +0,0 @@
# Facet Service RPC
Chord owns the application-neutral service semantics and pluggable strict-JSON connection
boundary. Pi owns the concrete wire envelope, routing, attachment state, and error adapters
described here. The current implementation treats `JsonValue` as a static contract and defers
runtime rejection of unsupported values to the concrete serializer.
> **Status:** Design specification for experimental facet-service RPC semantics.
## Role
`provide()`/`use()` and `provideMany().spawn()`/`observe()` are the facet system's hidden RPC. Facets share TypeScript service contracts, while the transport carries service/member identifiers, strict JSON values, request/subscription correlation, keyed generations, and binding control messages. Hosts construct the typed local implementation or facade; TypeScript types and arbitrary objects never cross the wire. Independently loaded processes may temporarily run different source generations, so their service contracts must remain forward compatible across the supported skew window; version negotiation remains deferred.
```text
session/server facet: provide() / provideMany().spawn()
↕ hidden service RPC
presentation/session facet: use() / observe()
```
The service system is the extension boundary. Presentation facets receive semantic services and replicated state; they never receive a raw Harness, Session, tool registry, hook registry, credential store, or storage handle.
## Non-goals
Do not serialize `Context`, `AbortSignal`, telemetry objects, callbacks, tools, hooks, functions, or arbitrary object graphs. Do not make core Harness or Session implementations aware of transport mechanics. Do not make disconnect perform durable or service-owned cancellation. Do not build per-method codecs for values already constrained to JSON.
## Service contracts and typed facades
A service token is a shared TypeScript contract and stable service ID. It is not a generated descriptor and creates no provider. Tokens are remotely publishable by default; process-local tokens declare `{ local: true }`. `provide()` adds one singleton implementation to the host graph. `provideMany()` registers one multi-instance service owner during facet setup and returns a `ServiceSpawner` whose later `spawn()` calls publish instances. The host automatically publishes every non-local provision. A token has one mode in one host service graph: mixing singleton and keyed use is an error.
The provider classifies every exposed implementation member as a method or Chord-created `ReplicatedState` and publishes that member table in subscription snapshots. The consumer obtains member names from ordinary property access—for example, a JavaScript `Proxy` receives `"state"` for `models.state` and `"refresh"` for `models.refresh(context)`. Accessed slots are validated against the provider-announced kind when the facade binds.
Local and remote `use()` both return a stable, lazy typed facade shared by consumers of that token. During synchronous facet setup the facade is disconnected, so setup can capture it but cannot invoke methods, read state, or register member subscriptions. After assembly, a local facade resolves through a direct process-local implementation slot and a remote facade binds through the host's connected services. Reloading the providing facet temporarily marks that same facade unavailable, then swaps its target; RPC singletons clear readiness and install a complete replacement snapshot on their existing subscription so captured methods and member facades address the replacement. While no provider is bound, invoking a method fails and state remains unhydrated; no call is queued merely because it was made through a proxy.
Remote methods return promises and accept and return strict JSON apart from their declared `Context`; `void` is a successful response without a result field. Private returned references are not supported. The client removes the context before transport and the receiving host constructs a fresh local context. The contract position is host-controlled and must be consistent; the examples use one required trailing `Context`. Business absence is JSON `null` or an options object, never transported `undefined`.
Use static assertions and runtime validation. Static checks constrain remote methods and replicated-state members; runtime boundaries reject unsupported members and non-JSON arguments, results, and state values. TypeScript supplies typed facades but does not authenticate a peer or create runtime metadata.
`{ local: true }` removes only remote publication and its wire-contract restrictions. Local and non-local provisions otherwise use the same dependency ledger, activation order, stable singleton slots, keyed-instance generations, observer cancellation, disposal, and provider-facet reload. Local singleton slots and the local keyed registry hold arbitrary object contracts directly; non-local provisions additionally install their implementations in the remote provider.
## Dependency ledger
Type erasure does not hide service identity: every service token retains its stable ID at runtime. Facet environments are created by the host with a non-forgeable owner identity, and their setup-time service methods append to a generation-scoped ledger:
- `provide()` records a singleton provision;
- `provideMany()` records a keyed provision;
- `use()` records a singleton requirement; and
- `observe()` records a keyed requirement.
Facets always call unqualified `env.use()` or `env.observe()`; routing is not encoded in the call. These operations return source-independent disconnected handles during setup. After setup, each connection returns its provider-generated service catalogue, and the host binds every requirement to its local provision or exactly one connected provider. The method supplies the mode and the token supplies the ID. The host therefore needs no reflection over the erased `T` and no handwritten parallel dependency list.
First acquisition or provision is permitted only during facet setup. Later commands, hooks, and activation callbacks use setup-acquired singleton facades, observer registrations, or `ServiceSpawner` capabilities. In particular, dynamic instances are spawned through the capability returned by `provideMany()`; late spawning cannot introduce a previously undeclared provision.
After setup, each host privately resolves the recorded requirements against local provisions and connection catalogues to reject missing providers, duplicate remote offers, and mode mismatches and to derive lifecycle edges. A selected-Session connection with no live attachment may provisionally accept unresolved requirements as unavailable; attachment validates them against the worker's generated catalogue and caches that catalogue for later detached generations. Connection bindings are generation-owned and include only selected requirements, so failed or retired generations release their subscriptions without disposing the underlying transport connection. This internal service graph is distinct from the module loader's source import graph; it is not a facet-authored or facet-visible plan.
## Bindings and identity
A presentation host combines services from its connected server and selected Session in one graph. All of its facets use the same unqualified environment API. A Session-service call never accepts a client-selected durable `sessionId`; the server authorizes and routes the presentation's selected Session binding to its worker.
### Server control plane
Session listing and management are ordinary server singleton services, not generic remote `Session` methods. `SessionDirectory` exposes presentation-safe session summaries as replicated state. `SessionManagement` exposes `create`, `remove`, `attach`, and `detach` methods. A TUI or web facet consumes both through its environment:
```ts
const directory = env.use(SessionDirectory);
const management = env.use(SessionManagement);
```
The server binds one service provider to each presentation connection. It derives workspace and client authority from locally authenticated connection identity, not from summary fields or method arguments. It may project directory state per client; either way, summaries never expose server-private fields such as owner IDs or working directories.
`management.attach(sessionId, context)` changes the selected-Session services in the presentation host. The server closes the presentation's previous Session-scoped requests, subscriptions, and observer tasks, binds the presentation's Session services to the worker, then hydrates their singleton state and keyed-instance directory. The server authorizes the selected Session against the connection identity. Attachment state is host control state reporting this selection and its health; it is not a directory service. `detach()` performs the same cleanup without a replacement.
The host needs a private, host-owned binding incarnation for that route. It changes when the presentation attaches, detaches, switches session, or replaces a failed worker. Its representation is deliberately unspecified. The binding prevents a delayed frame for the old selected session from being applied to the new one; it is not a facet-visible service value or a substitute for authorization.
A replicated-state source has structural identity:
```text
(provider binding, service ID, optional instance key + generation, member name)
```
There is no separately discoverable state ID. An added instance key is an application-level logical key. Its host-owned generation changes when a closed key is reused, so a stale proxy cannot call the replacement. `requestId` identifies one transport invocation for response and cancellation. A Harness/tool `invocationId` may be a useful instance key—as in the question example—but it does not replace the service, binding, or generation parts of the live address.
## Calls, context, and routing
A call carries enough control-plane information to select a provider binding, service, optional keyed instance, and member, plus a request ID, JSON arguments, and trace carrier. The server may parse those control-plane fields to route a Session call, but it does not parse facet business payloads or load facet contracts. The service endpoint validates the member and values, creates a request-local abort controller and `Context`, installs authenticated identity, and invokes the local implementation.
The client maps `context.abortSignal` to cancellation of that one request. Disconnect cancels that connection's active calls and closes its subscriptions. Neither action cancels service-owned work or writes durable Harness cancellation. Per-client request correlation reaches the worker so request IDs from different presentations cannot collide.
## Replicated state and keyed instances
`ReplicatedState` is authoritative latest-value replication, not event history, durable storage, a CRDT, or multi-writer state. A cold replica has `value === undefined`; subscribing before hydration records a listener without invoking it. Hydration installs a complete snapshot atomically before later updates are delivered, so there is no snapshot/update gap. Once hydrated, `value` is synchronous and subscribing reports the current value followed by later updates. The first snapshot callback uses a fresh hydration context; an already hydrated replica uses a fresh local delivery context rather than retaining the original write context. State values are borrowed immutable JSON and are not defensively cloned; callers must not mutate or retain them.
Remote hydration uses a fresh delivery context parented to the subscription, while updates reconstruct fresh delivery contexts from source trace metadata. Disconnect, provider withdrawal, replacement, and route switching clear readiness. Reconnect or replacement installs a complete snapshot before later updates. The transport buffers updates that race hydration and checks their sequence. Acknowledgements, flow control, and gap recovery remain separate protocol mechanics.
`observe()` is keyed-instance discovery, not a `ReplicatedState` containing proxies. It reconciles a complete initial directory with ordered additions, replacements, and removals. Each instance's initial state members hydrate before its observer task starts. Closing an instance rejects new calls, aborts only that instance's observer task, and allows admitted calls to settle. A Session facet's `env.observe()` registration aborts old tasks when that facet generation closes; the replacement generation reconciles the fresh directory.
## Private returned references
Private returned references are outside the initial service contract. Prefer keyed services for discoverable live instances. If a concrete feature requires caller-private remote identity, its reference must be passed explicitly rather than discovered by `observe()` and scoped to the recipient and provider binding.
No generic Harness projection is part of this design. Raw Harness, Session, lane, tool, hook, and storage objects remain local authority. If a future integration needs a remote callback or a general object capability, it needs a separate explicit protocol and policy; it is not an extension of service RPC.
## Context, cancellation, and telemetry
Every remote method receives a fresh local `Context`; the sender's object, signal, telemetry implementation, and arbitrary typed values never cross the wire. The client maps the call's abort signal to that request and injects a trace carrier. The endpoint constructs a request-local abort signal and telemetry parent. Cancellation is forwarded through the server to Session workers and remains isolated by client plus request ID.
The span relationship is:
```text
caller
└─ rpc.client
└─ rpc.server
└─ service implementation
```
Three cancellation domains remain separate: aborting one RPC invocation; explicitly cancelling service-owned work such as `job.cancel()`; and durable Harness cancellation such as `requestAbort()`. Transport cancellation and disconnect perform only the first. Work that outlives a call must detach into a service-owned task with its own controller and telemetry root.
## Security and lifecycle
Only loaded service tokens not marked local may be registered at the remote boundary. Only the owning `ServiceSpawner` may spawn instances. Services marked `{ local: true }` are never discoverable remotely. Local services may use unrestricted object contracts. Remote providers validate member kinds, while concrete serializers enforce JSON business values. Clients cannot forge control envelopes as ordinary values, choose instance generations, select a different Session route in a service call, or cancel another client's request.
The server authenticates connections, authorizes attachment, and reconstructs client identity in the service `Context`. Ordinary business arguments never carry authority. Credentials, prompts, completions, tool data, filesystem contents, and other sensitive values require an explicit presentation-safe contract.
Facet environments own registrations, added service instances, observations, and resources explicitly registered through `own()`. Connection bindings own their transport subscriptions and active request controllers. Already-admitted inbound calls are not attached to the providing facet lifecycle: provider withdrawal rejects new calls, but an admitted method may continue while the old facet deactivates. The provider's own Session work remains alive unless its lifecycle policy stops it.
## Tests
The facet-facing semantics are tested over loopback and framed transports:
- setup-time dependency-ledger ownership, rejection of late acquisition, local and remote `use()`, keyed-provider ownership, singleton/keyed mode validation, token-driven RPC publication, lazy member access, stable local and RPC singleton facades across provider-facet replacement, and `{ local: true }` services remaining unreachable remotely;
- strict JSON boundaries, method context reconstruction, and request cancellation isolation without serializing context values;
- server/Session facet isolation, selected-Session switching, stale-frame rejection, and worker-side per-client request correlation;
- cold and hydrated `ReplicatedState`, snapshot/update race freedom, fresh delivery contexts, and clearing/re-hydration on disconnect, reconnect, and provider replacement;
- instance directory hydration, ordered reconciliation, state hydration before observer tasks, generation-based stale rejection, and task cleanup on close or switch.
Additional tests cover authenticated attachment and identity, telemetry propagation, flow control and gap recovery, plus the question and shared-review application patterns. If private references are added, they require separate lifetime and isolation coverage.
## Open protocol mechanics
Exact service call, cancellation, subscription, snapshot/update, keyed-instance, unavailable, and replacement frames are defined in `packages/protocol/src/protocol.ts`. Provider and namespace layers own member classification, lazy facades, buffering, and sequencing.
Still open are acknowledgements, flow control, sequence-gap recovery, reference collection if references are added, protocol-version negotiation, and how a future multi-pane presentation represents more than one selected Session. Singleton provider replacement must continue to install a complete replacement snapshot on the existing subscription so method and state member slots retain identity.
## Example: directory and selected session
The directory and management services are normal server services. Their contracts carry only presentation-safe values:
```ts
interface SessionSummary {
serverId: string;
sessionId: string;
createdAt: string;
}
interface SessionDirectory {
readonly state: ReplicatedState<{ revision: number; sessions: SessionSummary[] }>;
}
interface SessionManagement {
create(options: { id?: string }, context: Context): Promise<SessionSummary>;
remove(sessionId: string, context: Context): Promise<void>;
attach(sessionId: string, context: Context): Promise<void>;
detach(context: Context): Promise<void>;
}
const SessionDirectory = defineService<SessionDirectory>("pi.session-directory");
const SessionManagement = defineService<SessionManagement>("pi.session-management");
```
A server facet derives the client from an authenticated `Context`, authorizes the requested Session, and performs the binding transition:
```ts
serverContext.provide(SessionDirectory, { state: directoryState });
serverContext.provide(SessionManagement, {
async attach(sessionId, context) {
const client = requireClientIdentity(context);
authorizeSession(client, sessionId);
await attachments.bind(client.clientId, sessionId, context);
},
async detach(context) {
await attachments.unbind(requireClientIdentity(context).clientId, context);
},
});
```
A presentation facet renders and selects Sessions:
```ts
setup(env) {
const directory = env.use(SessionDirectory);
const management = env.use(SessionManagement);
const tui = env.use(Tui);
tui.commands.register("sessions.switch", async (operation) => {
const snapshot = directory.state.value;
if (snapshot === undefined) return;
const sessionId = await tui.select(
"Sessions",
snapshot.sessions.map((session) => ({ label: session.sessionId, value: session.sessionId })),
{ signal: operation.abortSignal },
);
if (sessionId !== undefined) await management.attach(sessionId, operation);
});
}
```
Another facet in the same presentation acquires Session services through the same API:
```ts
setup(env) {
const models = env.use(Models);
// After attach() settles, `models` addresses the selected worker.
}
```
The presentation never routes `models` with a selected `sessionId`; its host routes each service token and transport retains the selected-Session binding. The server closes the prior Session binding's resources before hydrating the new one.

391
third_party/pi/packages/agent/docs/runtime-simplification.md vendored
View File

@@ -1,391 +0,0 @@
# Existing AgentHarness runtime simplification
## Scope
Rework the real implementation under `packages/agent/src/harness/runtime/` and the canonical durable types in `packages/agent/src/harness/session/types.ts`. This is not the isolated scratch spike.
The execution graph is total and public drive is enabled. Format 4 is still work in progress, so the durable type replacement required no migration or compatibility representation.
## Implementation status
Pre-M6 completion measurements:
- pre-simplification runtime at `eb1185d93`: 5,358 TypeScript lines;
- runtime after the first simplification at `417905647`: 4,667 lines;
- simplified substrate at `0e77e57d9`: 4,654 lines;
- total pre-M6 reduction: 704 lines (13.1%).
Implemented by the simplification pass:
- explicit `ContinueOperationResult<T>` with `cancel_requested` instead of implicit `undefined`;
- concrete generation, deferred-poll, and tool-call phases: prepare immutable input, publish durable intent, perform the effect, publish the durable outcome;
- shared assistant stream protocol consumption and Drive response lifecycle;
- `publishResponse` for common assistant/deferred durable publication and `publishConfigurationFailure` for their identical pre-intent failure transition;
- source-order tool publication, one-read immutable source validation, and shared tool-batch reconstruction isolated in `runtime/drive/tool-placement.ts`;
- same-call tool status defenses, unchecked `toolOperation()` casts, and their result unions removed; genuine sibling/progress/memo checks retained;
- centralized lane-storage classification without duplicate projection reads;
- checkpoint dead-check removal and exact checkpoint preservation;
- shared transcript mechanics in `runtime/transcript.ts`;
- narrower `LanePatch`, no redundant Drive promise-settlement flag, and no unused operation reject arm.
At simplification completion, the largest files were `runtime/lane.ts` (~996), `runtime/drive/tools.ts` (~663), `runtime/drive/checkpoint.ts` (~452), and `runtime/drive/response.ts` (~417).
### WP05 structural status
M6 established structural execution on the simplified substrate. Its first coherent slice added:
- shared compaction-bounded transcript/context reads and committed-entry event decoration;
- caller-owned one-provider-request seams for compaction and branch summaries while preserving retrying behavior for existing non-harness callers;
- direct procedures for every structural `at` leaf in `runtime/drive/structural.ts`;
- once-per-trigger threshold routing and atomic overflow preparation publication;
- per-request structural intent and usage settlement, attempt retry/recovery, hook decisions, compaction/navigation publication, and terminal cleanup;
- focused coverage for threshold/overflow entry, split-turn request accounting, generated and hook results, retry/cap/recovery, model absence, mid-request durable cancellation, navigation, and preparation corruption.
R1a, R2, and R3 then moved queued input to the lane, replaced hydrated family outcomes with immutable operation records, and collapsed the family cross-product to 13 neutral leaves. R1b replaced transitional checkpoint drains and marker fields with one shared atomic boundary planner, a transcript-derived threshold guard, and off-line finish-hook replanning. M7 removed drained control, added atomic drain-and-return cancellation, reconciled all 13 leaves, and installed the total direct dispatcher.
M8 added primitive/convenience lane surfaces, one ordered tagged inbox, idle ownership, snapshot/event replication, `reduceLaneSnapshot`, and remote resnapshot. M9 reconciled the normative specification. M10 forwards exactly one stable ordinary-provider identity derived from Session metadata id plus lane name; structural summaries keep fresh request identities. At M10 completion the runtime is 7,410 lines, `runtime/drive` is 4,147, `lane.ts` is 1,992, `structural.ts` is 1,221, and the new normative reducer is 193 lines. Relative to the reviewed M7 checkpoint, the public/replication surface added 1,130 runtime lines; the durable 13-leaf procedure model stayed unchanged.
Keep the visible durable procedure order:
`prepare → publish intent → perform effect → publish outcome`.
Do not introduce a generic Procedure interface, runner, scheduler, graph, callback plan, or dependency facade. Historical work-package documents remain unchanged.
## Core model
A Lane is one process-local actor over one durable lane projection.
- `Lane.state` is authoritative for tip, configuration, current operation, control, inbox IDs, and the latest operation ID.
- Every supported mutation commits through the Session mutation line and publishes the matching `Lane.state` before releasing it.
- One lane-owned Drive is the sole writer that advances an operation state.
- Same-operation callers observe the same Drive. No caller owns it.
- Invocation cancellation stops only that caller's observation after Drive installation.
- `requestAbort` is the only durable operation cancellation.
- Close seals mutation admission. It neither mutates operation state nor replaces a Drive.
- Process loss destroys all live continuations; recovery starts from the durable state after attachment.
- Session reads during execution dereference content named by `Lane.state`; they never rediscover control state.
## Durable state
`OperationState` has one discriminator, `at`, with 13 direct leaves:
- `starting`
- `checkpoint`
- `assistant.ready`
- `assistant.effect_pending`
- `assistant.retry_wait`
- `tools`
- `deferred.suspended`
- `deferred.effect_pending`
- `summary.deciding`
- `summary.ready`
- `summary.effect_pending`
- `summary.retry_wait`
- `navigation.ready_to_commit`
Every leaf carries one uniform `OperationScope`. The summary quadruple carries one `SummaryTask`; its closed `ResultBoundary` selects in-run checkpoint resumption, standalone finish, or navigation commit. The summary algorithm is derived from that boundary rather than duplicated in state. `Control` remains orthogonal. `ToolBatch` remains a child state machine because parallel tool children genuinely mutate sibling call statuses concurrently.
## State and content boundary
Procedures never read these addresses to decide execution:
- `laneState`
- `operationMeta`
- `operationState`
- `branchTip`
- `laneConfig`
- `operationResult`
They use the current `Lane.state` supplied by the mutation line.
Storage reads remain only for content and cleanup:
- prompt, assistant, deferred-source, final-assistant, and completed-tool entries;
- compaction-bounded branch context;
- `pendingEntry` payloads;
- assistant-frame lists;
- tool arguments, memos, and checkpoints;
- structural preparations;
- staged tool outcomes;
- operation-owned prefix scans required for terminal cleanup.
## Concurrency model
### Operation advancement
While a Drive procedure is alive, no supported concurrent actor can change or remove its operation state:
- inbox methods change only inbox fields;
- `requestAbort` changes only `control`;
- close prevents later mutation admission;
- another same-operation caller joins the existing Drive;
- another operation cannot be accepted while the lane is busy;
- process crash removes the continuation itself.
Consequently, ordinary procedure transitions do not recheck operation existence, operation ID, operation kind, `at`, attempt identity, or nested status. Those are procedure preconditions established by dispatch, not supported races.
The remaining real races are:
1. cancellation arriving before effect admission or before settlement;
2. inbox arrival before checkpoint routing or terminal finish;
3. parallel tool children staging and materializing sibling outcomes;
4. queued frame/checkpoint/memo writes racing effect settlement;
5. retry timers racing cancellation or close;
6. deferred permit consumption;
7. accept/claim serialization on the Session line.
### Cancellation boundaries
Cancellation is checked in three places only:
1. the drive loop before ordinary dispatch;
2. the gate immediately before an external effect starts;
3. effect settlement, which sees current `control` and commits the appropriate cancelled result.
An ordinary transition helper may centrally decline progress when current control is `cancel_requested`. Procedures do not repeat that branch manually.
### Close
Closing the harness:
1. marks harness/lane admission closed;
2. seals and drains the Session mutation line;
3. rejects client observations through a harness-close observation promise;
4. keeps every detached pass promise observed so a later rejection is never unhandled;
5. closes the Session after admitted mutations drain.
Close performs no durable write, installs no replacement Drive, and creates no ownership-loss state. A late effect may return, but its mutation is rejected with `HarnessClosed`.
Whether close also signals process-local provider/tool work is a resource-cleanup policy, not durable state-machine behavior. Do not couple it to Drive replacement or recovery.
## Small concrete mutation API
Replace repeated `LaneCommand` ceremony in procedures with two concrete Lane operations. They are not a scheduler, graph, or action interpreter.
### `continueOperation`
Used for ordinary non-terminal progress.
- Enters the Session mutation line.
- Receives the current authoritative Lane projection and operation state.
- If control is cancelled, returns explicit `cancel_requested` without invoking the semantic planner; callers cannot confuse cancellation with a planner value.
- The planner supplies procedure-specific writes, the next complete `OperationState`, materialization, and events.
- The helper appends the `operationState` write and publishes the matching process-local operation projection.
- It does not verify an expected state or `at` value.
### `settleOperation`
Used after an admitted provider/tool/structural effect and for genuine parallel child transitions.
- Enters the Session mutation line even when control is cancelled.
- Supplies current control/inbox fields and the process-local effect result to the semantic settlement planner.
- Atomically commits payload, usage, tip movement, cleanup, and the classified next state.
- Appends the canonical operation-state write and publishes the matching projection.
- When the planner returns a terminal decision, appends the immutable `operationResult`, idle `laneState` with `lastOperationId`, and the idle process-local projection.
The caller supplies the typed outcome, cleanup/publication writes, last result, and event. Terminal business decisions remain visible in the owning procedure.
## Durable procedure shape
Effectful procedures expose four concrete phases:
```text
prepare immutable inputs
→ publish durable effect intent
→ perform the external effect
→ publish one durable outcome
```
The intent phase must remain visible and precede the external effect. Otherwise a crash after the effect but before its intent would leave no recoverable unknown-outcome marker. Each procedure uses concrete functions such as `prepareGeneration`, `publishGenerationIntent`, `performGeneration`, and `publishResponse`; there is no generic Procedure abstraction.
## Drive lifecycle
Delete the installer-owned model.
- Remove `installerSignal`.
- Remove `DriveAbandoned`.
- Remove `LostOwnership` and `lost_ownership` from procedure results.
- Remove exact-object ABA fencing and `commandDriveOwned`.
- Remove `finalizedOutcome` and planned external-finalization owner retention unless the Flue investigation establishes a concrete requirement.
- Keep `activeDrive` only as the lane's install/join slot.
- Installation transfers work to the lane. Every caller, including the installer, observes completion with its own invocation Context.
- A caller signal abort before installation installs nothing. After installation it only rejects that caller's observation.
- The Drive is removed after its pass settles or reaches a durable wait. No live pass is replaced in-process.
`requestAbort` retains the two-part gate ordering:
```text
beginAbort before cancellation mutation
commit cancel_requested
signalAbort after commit
```
This prevents a new effect from entering while the durable marker is being committed.
## Checks that remain
Do not remove validation of external or referenced content:
- required entry existence and role;
- pending payload kind;
- deferred handle identity;
- provider stream protocol ordering;
- response stop-reason invariants;
- UUIDv7 follower timestamp parsing;
- configured model/tool availability;
- tool-call source index and staged result identity;
- parallel tool call status and ready-prefix placement;
- progress/memo invocation identity;
- retry timestamp and deferred permit arithmetic.
These validate data or genuine child concurrency. They are not defensive revalidation of the operation state machine.
## Completed staged implementation plan
### Stage 1 — Canonical flat durable types
Files:
- `src/harness/session/types.ts`
- compile-only consumers in `src/harness/runtime/`, restore, conformance helpers, and focused tests
- `docs/harness.md`
- `docs/work-packages/05-direct-durable-drive.md`
Actions:
- Replace nested operation state declarations with the flat `at` union.
- Preserve every durable datum without compatibility aliases.
- Update pattern matching mechanically without changing behavior.
- Keep ToolBatch/ToolCall nested.
- Update normative documentation in the same change.
Exit: `npm run check`; existing focused runtime tests pass; no `phase.kind`, generation `status`, deferred `status`, or structural decision `status` remains in canonical operation state.
### Stage 2 — Add canonical transition operations
Files:
- `src/harness/runtime/lane.ts`
- `src/harness/runtime/types.ts`
- focused Lane tests
Actions:
- Add `continueOperation` and `settleOperation`, including the terminal-decision suffix.
- Pair every durable operation-state write with process-local projection publication in one implementation.
- Centralize ordinary cancellation diversion.
- Trust the dispatcher-established current leaf; perform no expected-state check.
- Keep procedure-specific writes and event builders visible at call sites.
Exit: focused tests prove durable state and `Lane.state` remain identical after every helper commit.
### Stage 3 — Convert starting, checkpoint, and assistant
Files:
- `runtime/drive/checkpoint.ts`
- `runtime/drive/generation.ts`
- `runtime/drive/recovery.ts`
- `runtime/progress.ts`
Actions:
- Remove repeated operation/null/kind/state checks and `same*` predicates.
- Convert ordinary progress to `continueOperation`.
- Convert assistant settlement to `settleOperation`.
- Keep only checkpoint inbox/finish races and progress-channel ownership checks.
- Fold assistant recovery into the effect-pending handler if that is smaller.
Exit: no control-state storage reads; no repeated assistant state verification; focused generation tests pass.
### Stage 4 — Convert deferred and tools
Files:
- `runtime/drive/deferred.ts`
- `runtime/drive/tools.ts`
- `runtime/progress.ts`
Actions:
- Share assistant/deferred response-entry, usage, and tool-plan construction where their semantics match.
- Preserve deferred permit and handle checks.
- Remove top-level operation-state revalidation.
- Preserve per-call tool status merging, completion-order staging, source-order placement, memo fencing, and progress fencing.
- Use one tool-batch procedure for live, recovery, and cancellation modes instead of separate ownership-result paths.
Exit: tool status checks exist only for genuine sibling concurrency; focused deferred/tool/progress tests pass.
### Stage 5 — Remove ownership-loss machinery
Files:
- `runtime/types.ts`
- `runtime/lane.ts`
- `execution/effect-gate.ts`
- all existing `runtime/drive/*.ts`
- associated focused tests
Actions:
- Remove `LostOwnership`, `commandDriveOwned`, exact Drive checks, `installerSignal`, `DriveAbandoned`, and `finalizedOutcome`.
- Make all drive callers observation peers.
- Add observation-only Context cancellation to Drive completion waiting.
- Keep `activeDrive` only for install/join arbitration.
- Remove ABA/replacement tests and replace them with install/join/observation-cancellation tests.
Exit: grep finds no ownership-loss or installer-abandonment vocabulary in production runtime.
### Stage 6 — Close and fault
Files:
- `runtime/harness.ts`
- `runtime/lane.ts`
- Drive observation helper
- lifecycle tests
Actions:
- Seal mutation admission and drain admitted mutations.
- Reject client observations on close/fault without replacing a Drive or changing durable operation state.
- Observe detached pass failures.
- Verify a late effect cannot commit after close.
- Decide separately whether to signal local effects for resource cleanup.
Exit: close and process loss leave the same durable restart point; no close path writes cancellation or synthetic settlement.
### Stage 7 — Finish WP05 on the simpler substrate
Files:
- `runtime/drive/structural.ts`
- `runtime/drive/reconcile.ts`
- `runtime/drive.ts`
- `runtime/lane.ts` public surfaces
Actions:
- Implement structural generation directly over flat leaves and the transition operations.
- Implement cancellation reconciliation as one flat-state switch.
- Implement the total drive switch as one `state.at` switch.
- Wire public claim/join/observation and convenience methods only after every leaf is total.
External finalization is excluded unless the Flue investigation identifies a concrete, current caller that cannot be expressed through close, explicit abort, recovery, or offline administration.
## Validation
After every code stage:
```bash
npm run check
```
Run each modified focused test file from the package root. Do not run the full Vitest suite directly. Public drive was enabled only after the final stage made every leaf total.
Final audit:
```bash
rg 'lost_ownership|LostOwnership|DriveAbandoned|commandDriveOwned|installerSignal|finalizedOutcome' packages/agent/src/harness
rg 'operationState\(|laneState\(|branchTip\(|laneConfig\(' packages/agent/src/harness/runtime/drive
```
The second audit may match write constructors, but no reader call may use those control addresses.

383
third_party/pi/packages/agent/docs/telemetry-schema.md vendored
View File

@@ -1,383 +0,0 @@
# Pi Agent Telemetry Schemas
<!-- Generated by generate-telemetry-docs.ts. Do not edit manually. -->
## AI request schema
Schema version: 1
### `pi.ai.request`
One logical request to an AI provider
- Parents: root or any caller span
- Default status: `ok`
- Error when: The operation throws or returns an error result
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.ai.operation` | `string` | yes | stream, fetch_deferred, cancel_deferred, generate_images | | Logical provider operation |
| `pi.ai.provider` | `string` | yes | | | Selected provider id |
| `pi.ai.model` | `string` | yes | | | Requested model id |
| `pi.ai.api` | `string` | yes | | | Provider API id |
| `pi.ai.streaming` | `boolean` | yes | | | Whether this operation returns a stream |
| `pi.ai.deferred` | `boolean` | no | | | Whether the operation requests or participates in deferred execution |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.ai.response.model` | `string` | | | Concrete response model |
| `pi.ai.response.id` | `string` | | high cardinality | Provider response id |
| `pi.ai.response.stop_reason` | `string` | stop, length, tool_use, error, aborted, deferred | | Normalized terminal response reason |
| `pi.ai.http.status_code` | `number` | | | Final HTTP status |
| `pi.ai.usage.input_tokens` | `number` | | | Reported input tokens |
| `pi.ai.usage.output_tokens` | `number` | | | Reported output tokens |
| `pi.ai.usage.cache_read_tokens` | `number` | | | Reported cache-read tokens |
| `pi.ai.usage.cache_write_tokens` | `number` | | | Reported cache-write tokens |
| `pi.ai.usage.reasoning_tokens` | `number` | | | Reported reasoning tokens |
| `pi.ai.usage.total_tokens` | `number` | | | Reported total tokens |
| `pi.ai.usage.cost` | `number` | | | Reported total cost |
| `pi.ai.stream.chunk_count` | `number` | | | Streamed update chunk count |
| `pi.ai.stream.time_to_first_chunk_ms` | `number` | | | Elapsed milliseconds to first update chunk |
| `pi.ai.error.type` | `string` | | low cardinality | Provider or transport error class |
#### Events
No declared span events.
## Harness schema
Schema version: 1
### `pi.harness.run`
One admitted in-process run invocation
- Parents: root or caller-owned external span
- Default status: `ok`
- Error when: The run fails or throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.session.id` | `string` | yes | | high cardinality | Session id |
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.operation.recovery` | `boolean` | yes | | | Whether this invocation resumes durable work |
| `pi.operation.kind` | `string` | yes | run | | Run operation kind |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.operation.outcome` | `string` | completed, aborted, failed, suspended | | Run invocation outcome |
| `pi.error.code` | `string` | | low cardinality | Stable operation error code |
| `pi.error.type` | `string` | | low cardinality | Low-cardinality operation error class |
#### Events
No declared span events.
### `pi.harness.compaction`
One admitted in-process manual compaction invocation
- Parents: root or caller-owned external span
- Default status: `ok`
- Error when: The compaction fails or throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.session.id` | `string` | yes | | high cardinality | Session id |
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.operation.recovery` | `boolean` | yes | | | Whether this invocation resumes durable work |
| `pi.operation.kind` | `string` | yes | compaction | | Compaction operation kind |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.operation.outcome` | `string` | completed, declined, aborted, failed | | Compaction invocation outcome |
| `pi.error.code` | `string` | | low cardinality | Stable operation error code |
| `pi.error.type` | `string` | | low cardinality | Low-cardinality operation error class |
#### Events
No declared span events.
### `pi.harness.navigation`
One admitted in-process navigation invocation
- Parents: root or caller-owned external span
- Default status: `ok`
- Error when: The navigation fails or throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.session.id` | `string` | yes | | high cardinality | Session id |
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.operation.recovery` | `boolean` | yes | | | Whether this invocation resumes durable work |
| `pi.operation.kind` | `string` | yes | navigation | | Navigation operation kind |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.operation.outcome` | `string` | completed, declined, aborted, failed | | Navigation invocation outcome |
| `pi.error.code` | `string` | | low cardinality | Stable operation error code |
| `pi.error.type` | `string` | | low cardinality | Low-cardinality operation error class |
#### Events
No declared span events.
### `pi.harness.checkpoint`
One run checkpoint
- Parents: `pi.harness.run`
- Default status: `ok`
- Error when: Checkpoint work throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.checkpoint.kind` | `string` | yes | normal, abort_reconcile | | Checkpoint purpose |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| _none_ | | | | |
#### Events
No declared span events.
### `pi.harness.turn`
One assistant response and its tool batch
- Parents: `pi.harness.run`
- Default status: `ok`
- Error when: Turn work throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.turn.id` | `string` | yes | | high cardinality | Invocation-local turn id |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| _none_ | | | | |
#### Events
No declared span events.
### `pi.harness.step`
One durable retry attempt
- Parents: `pi.harness.turn`, `pi.harness.checkpoint`, `pi.harness.compaction`, `pi.harness.navigation`
- Default status: `ok`
- Error when: The attempt retries, fails, or throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.step.kind` | `string` | yes | assistant, compaction, branch_summary | | Retryable step kind |
| `pi.step.attempt` | `number` | yes | | | One-based durable attempt number |
| `pi.compaction.reason` | `string` | no | manual, threshold, overflow | | Compaction trigger |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.step.outcome` | `string` | succeeded, retry, failed, aborted, deferred, overflow | | Attempt outcome |
#### Events
No declared span events.
### `pi.harness.tool`
One raw phase-2 tool execution
- Parents: `pi.harness.turn`, `pi.harness.run`
- Default status: `ok`
- Error when: Raw phase-2 execution returns an error
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.turn.id` | `string` | no | | high cardinality | Invocation-local live turn id |
| `pi.tool.name` | `string` | yes | | | Tool name |
| `pi.tool.call_id` | `string` | yes | | high cardinality | Tool call id |
| `pi.tool.replay` | `string` | yes | never, safe | | Declared replay policy |
| `pi.tool.recovery` | `boolean` | yes | | | Whether this is recovery execution |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.tool.is_error` | `boolean` | | | Whether raw phase-2 execution returned an error |
#### Events
No declared span events.
### `pi.harness.hook`
One registered hook handler invocation
- Parents: root or any caller span
- Default status: `ok`
- Error when: The handler throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.lane.name` | `string` | yes | | high cardinality | Lane name |
| `pi.operation.id` | `string` | no | | high cardinality | Durable operation id when accepted |
| `pi.hook.name` | `string` | yes | before_run, before_drive, before_run_end, transform_context, before_request, before_payload, after_response, before_tool, after_tool, before_compaction, before_navigation | | Hook name |
| `pi.hook.registration_id` | `string` | no | | | Optional hook registration metadata |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.hook.outcome` | `string` | completed, skipped, blocked, failed | | Handler outcome |
#### Events
No declared span events.
### `pi.harness.sleep`
One retry delay
- Parents: `pi.harness.run`, `pi.harness.compaction`, `pi.harness.navigation`, `pi.harness.turn`, `pi.harness.checkpoint`
- Default status: `ok`
- Error when: Sleep work throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id |
| `pi.sleep.delay_ms` | `number` | yes | | | Requested delay in milliseconds |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.sleep.outcome` | `string` | elapsed, aborted | | Delay outcome |
#### Events
No declared span events.
### `pi.harness.event_handler`
One passive event listener invocation
- Parents: root or any caller span
- Default status: `ok`
- Error when: The listener throws
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.event.type` | `string` | yes | run_start, run_resume, run_suspend, operation_abort, run_end, fault, handler_error, turn_start, turn_end, retry_scheduled, retry_start, retry_end, message_start, message_update, message_end, tool_start, tool_update, tool_end, entry_added, queue_update, value_update, config_update, compaction_start, compaction_end, navigation_start, navigation_end, lane_created, usage | low cardinality | Delivered harness event type |
| `pi.lane.name` | `string` | no | | high cardinality | Lane name for lane-scoped events |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| _none_ | | | | |
#### Events
No declared span events.
### `pi.session.write`
One committed session transaction
- Parents: root or any caller span
- Default status: `ok`
- Error when: Storage rejects the transaction
#### Start attributes
| Name | Type | Required | Values | Notes | Description |
|---|---|---:|---|---|---|
| `pi.session.id` | `string` | yes | | high cardinality | Session id |
| `pi.lane.name` | `string` | no | | high cardinality | Lane name when supplied by the caller |
| `pi.operation.id` | `string` | no | | high cardinality | Durable operation id when supplied by the caller |
| `pi.session.item_count` | `number` | yes | | | Number of writes in the transaction |
| `pi.session.item_kinds` | `string[]` | yes | elements: entry, usage, value, list | | Distinct write kinds in the transaction |
#### End attributes
All end attributes are optional completion enrichment.
| Name | Type | Values | Notes | Description |
|---|---|---|---|---|
| `pi.session.first_seq` | `number` | | | First committed sequence in the transaction |
| `pi.session.last_seq` | `number` | | | Last committed sequence in the transaction |
#### Events
No declared span events.

251
third_party/pi/packages/agent/docs/telemetry.md vendored
View File

@@ -1,251 +0,0 @@
# Invocation Context and Telemetry Design Notes
> **Status:** Design input, not a normative contract. The context primitives and required trailing `Context` parameters have landed across the harness, sessions, execution capabilities, and hosted-harness adapters. Local propagation is scaffolding rather than proof of complete telemetry semantics: most runtime spans and cross-process trace propagation remain design or implementation work. Drive ownership is now harness-owned and request-ID RPC cancellation is implemented; neither supplies distributed trace parentage. Fold accepted final behavior into `harness.md`. `telemetry-schema.md` remains the generated reference for span names and attributes.
## Goal
`Session`, `Branch`, `AgentLane`, and `AgentHarness` receive invocation-scoped control data explicitly through a required trailing `Context` parameter. The same receiver may serve concurrent local callers or RPC clients, so it cannot retain a mutable or default caller context.
The invocation context must solve two related problems without `AsyncLocalStorage`:
1. preserve correct telemetry parentage through concurrent asynchronous work;
2. carry an `AbortSignal`, when one exists, that an RPC adapter can map to request cancellation.
This work must reuse `@earendil-works/pi-telemetry`. It must not introduce another span abstraction.
## Context model
The implemented public types are:
```ts
interface ContextKey<T> {
readonly token: symbol;
readonly valueType?: (value: T) => T;
}
interface Context {
readonly abortSignal: AbortSignal | undefined;
readonly telemetryContext: TelemetryContext;
value<T>(key: ContextKey<T>): T | undefined;
toString(): string;
}
```
`valueType` is a type-only marker. Runtime lookup uses the key's symbol token. `createContextKey<T>(description)` creates and freezes a key with a unique token.
A context is immutable. Derivation creates a parent-linked, copy-on-write layer. Helper arguments put the value first and parent context last:
```ts
const requestContext = withAbortSignal(requestSignal, parentContext);
const spanContext = withTelemetryContext(span, requestContext);
const tenantContext = withContextValue(tenantKey, tenantId, spanContext);
```
The implemented behavior is:
- `BACKGROUND_CONTEXT` and `TODO_CONTEXT` are distinct empty roots whose `abortSignal` is `undefined`;
- `telemetryContext` is always available and falls back to `NOOP_TELEMETRY_CONTEXT` when no telemetry value has been installed;
- `withAbortSignal(signal, context)` preserves the supplied signal when the parent has none and otherwise combines it with the parent signal using `AbortSignal.any()`;
- `withCancel(context)` returns an independently cancellable child context and a `cancel(reason?)` function; parent cancellation still reaches the child;
- typed values use symbol identity and immutable copy-on-write layers, and a newer value for the same key shadows its parent value;
- the built-in abort-signal and telemetry keys are private; callers use the named properties instead of retrieving those values by key;
- `toString()` is diagnostic and records the root plus each layered key description.
Context values are cross-cutting request metadata, not business dependencies. Suitable typed values include request IDs, authenticated principals, tenant IDs, and diagnostic metadata. Storage, models, tools, durable state, and business payloads do not belong in the context. Contexts, signals, telemetry objects, and backend-native span objects are never durable data.
## Receiver ownership
Shared receivers retain identity and durable/process state, not invocation context:
```text
AgentHarness receiver ── no caller context
AgentLane receiver ── no caller context
Session receiver ── no caller context
Branch receiver ── no caller context
```
Every invocation supplies its own context. This prevents concurrent callers from overwriting each other's telemetry parent or cancellation signal.
A process-local object representing one ongoing invocation may retain its derived context. Examples are an active drive task or an event subscription. This is different from storing a default context on the shared harness or session receiver.
`AgentHarnessOptions.telemetryContext` has been removed. A harness-level default cannot represent two concurrent callers with different parents.
## Existing typed telemetry remains authoritative
The design retains:
- `TelemetryContext` and `TelemetrySpan`;
- callback-owned span lifetime;
- `AI_TELEMETRY_SCHEMA` and `HARNESS_TELEMETRY_SCHEMA`;
- typed span names, start attributes, completion attributes, and events;
- `startAiSpan()`, `startHarnessSpan()`, and `createTypedSpanStarter()`;
- adapter conformance behavior.
`startAiSpan()` and `startHarnessSpan()` package span derivation by giving their callbacks both the typed span and a derived invocation context:
```ts
return startHarnessSpan(
"pi.harness.run",
attributes,
async (span, runContext) => {
return runDrive(runContext);
},
context,
);
```
The helpers delegate to `context.telemetryContext.startSpan()` and install the callback-owned span in a child context with `withTelemetryContext(span, context)`. Lower work must receive that child context rather than the parent invocation context.
Do not mutate a context to install an active span. Do not use a process-global or receiver-global current span.
## Concurrent parentage
Explicit propagation supports concurrent sibling calls:
```ts
await parent.telemetryContext.startSpan({ name: "caller" }, async (callerSpan) => {
const callerContext = withTelemetryContext(callerSpan, parent);
await Promise.all([
laneA.drive(optionsA, callerContext),
laneB.drive(optionsB, callerContext),
]);
});
```
Each invocation derives its own child context. Nested work receives the child belonging to that invocation. Correct parentage does not depend on promise scheduling or ambient state.
Tests must cross concurrent branches deliberately so accidental receiver-level context is visible. A sequential parent/child test is insufficient.
## Callbacks, hooks, and events
Host-local callbacks invoked as part of an operation receive the current invocation context in their declared trailing position:
```ts
handler(event, context);
tool.execute(toolCallId, params, onUpdate, toolContext, invocation, context);
mutation(mutator, context);
```
Current propagation preserves context through callbacks and gives `before_tool` and `after_tool` handlers a child context derived from `pi.harness.hook`. Extending that span behavior to every hook type remains work; handlers without an installed hook span currently receive the operation context directly.
Within the harness process, events preserve the context that caused each event, and buffered event watchers store `{ event, context }` rather than only `event`. Starting `pi.harness.event_handler` from that event context and passing its child context to each listener remains work. Event registration itself is host-local configuration and has no operation parent.
Session mutation callbacks and commits receive the same explicit invocation context. Starting `pi.session.write` from the committing invocation and passing its child context through the storage commit remains work.
## Drive execution and joiners
Several callers may call `drive()` for the same durable operation. Arbitration decides which call installs process-local execution and which calls join it. This is a core runtime concern, not an RPC concern; concurrent local callers have the same issue.
One active execution has one telemetry parent. It cannot be reparented when another caller joins.
```text
installer caller
└─ drive.execute
└─ provider/tool work
joiner caller
└─ drive.join
```
A joiner span describes that caller's wait. It carries at least lane name, durable operation ID, and a process-local execution ID. It ends with an outcome such as `settled`, `caller_cancelled`, `execution_stopped`, or `harness_closed`.
The joiner must not overwrite the active execution context. Correlate the two spans using operation/execution attributes. Telemetry span links would model this relationship better, but the current telemetry contract has no links. Adding links is an optional telemetry-package design question, not a reason to invent multiple parents.
Distributed traces permit an execution span to outlive the installer RPC span. Parent and child spans may overlap or settle in either order once the child has started.
## Invocation cancellation versus durable cancellation
An aborted invocation signal is process-local control. It does not mean that durable cancellation was requested.
```text
context.abortSignal is present and aborts
→ stop only that caller's observation; an installed lane-owned Drive continues
→ do not write cancel_requested
→ preserve the same durable operation state
```
An undefined `abortSignal`, as exposed by both empty roots, means that the invocation has no cancellation signal.
Only `requestAbort()`/`abort()` writes durable `cancel_requested` and permits durable aborted settlement.
The runtime must track the stop cause instead of interpreting every aborted provider response as durable cancellation:
```ts
type ExecutionStopCause =
| "no_drive_waiters"
| "invocation_cancelled"
| "harness_closed"
| "durable_cancel_requested";
```
Only `durable_cancel_requested` may normalize and commit a durable aborted outcome. An invocation/disconnect abort must not produce an assistant `stopReason: "aborted"` settlement while durable control remains `running`; that path would violate the durable state machine.
Drive ownership is resolved as **harness-owned**: once installed, execution survives caller cancellation/disconnect until durable settlement or wait, explicit durable cancellation, close, fault, or process loss. Joiner signals control only their own observations. Signals from unrelated joiners must never be combined with `AbortSignal.any()` and attached directly to shared execution. One canceled joiner cannot cancel every other caller.
## RPC trace propagation
Client and server spans can belong to one distributed trace:
```text
caller
└─ rpc.client
└─ rpc.server
└─ harness/session operation
```
The client does not serialize `TelemetryContext`. It injects a backend-neutral trace carrier from the `rpc.client` span. The server extracts that carrier into a fresh local `TelemetryContext` and starts `rpc.server` from it.
A transport-facing adapter boundary is required:
```ts
interface TelemetryPropagation {
inject(context: TelemetryContext): JsonValue | undefined;
extract(carrier: JsonValue | undefined): TelemetryContext;
}
```
A production implementation may use W3C `traceparent`/`tracestate`. The current telemetry package has no carrier injection/extraction API, so the accepted design must decide whether this adapter belongs in the telemetry package, RPC infrastructure, or a backend integration package. It must still reuse the existing `TelemetryContext` span contract.
RPC cancellation and telemetry propagation are independent control-plane channels:
- trace metadata reconstructs telemetry parentage;
- request ID plus cancel/disconnect messages controls the server request signal;
- neither channel appears in serialized method arguments.
## Interface migration scaffolding
Receiver methods now use a required trailing `Context`. Concrete implementations, calls, callback adapters, and object-literal façades have been migrated rather than relying only on interface assignability.
`TODO_CONTEXT` remains a temporary migration marker, not a semantic root. Current uses cluster at unresolved transport and worker boundaries that cannot yet reconstruct a caller context, notably Pi protocol request ingress and worker RPC ingress. `BACKGROUND_CONTEXT` means intentionally start without a caller.
Continue to inventory `TODO_CONTEXT` separately. Replace each transport-boundary use only when the boundary can construct a request-local cancellation context and telemetry parent; substituting `BACKGROUND_CONTEXT` would hide unfinished propagation. Compilation still does not prove telemetry or cancellation correctness.
## Required tests for the later handoff
Current tests cover immutable typed-value layering and shadowing, distinct empty roots, parent/child abort composition, sibling cancellation isolation, and tool-hook child parentage. Remaining handoff coverage includes:
- crossed concurrent telemetry branches on one shared receiver;
- every hook type, tools, event handlers, and session writes receive the intended child context;
- buffered events retain their emitting context under delayed delivery;
- no receiver-level telemetry default;
- pre-aborted invocation starts no external effect;
- installer and joiner cancellation isolation under lane-owned execution;
- invocation abort leaves the installed Drive and durable state unchanged;
- durable abort commits the durable aborted outcome;
- close and disconnect do not masquerade as durable cancellation;
- client → server trace reconstruction;
- event delivery reconstructs source trace metadata;
- missing/malformed trace carriers degrade to no-op/root telemetry without affecting business behavior.
## Resolved migration decisions
- Receiver methods use one required trailing `Context`.
- Shared Harness, AgentLane, Session, and Branch receivers retain no default invocation context.
- `Context`, `AbortSignal`, and `TelemetryContext` objects are never serialized across RPC boundaries.
## Open decisions before the telemetry handoff
- whether telemetry links are required for joiners;
- trace-carrier adapter ownership and shape;
- which context values, if any, may cross an RPC boundary;
- exact span names/outcome attributes for RPC calls and drive join waits.

704
third_party/pi/packages/agent/docs/tool-durability.md vendored
View File

@@ -1,704 +0,0 @@
# Tool durability — implementation handoff
This document specifies the durable tool-call lifecycle and the minimal harness-specific progress-checkpoint extension to the current tool API. It does not otherwise finalize the harness-native public tool interface; that interface must provide the capabilities required here without exposing raw session storage.
The design has two independent additions:
1. a durable `outcome_ready` state between external-effect settlement and source-ordered conversation placement;
2. opt-in durable replacement checkpoints for complete bounded `onUpdate` snapshots.
## Problem
Parallel tool effects finish in completion order, while tool-result entries must enter the conversation in assistant source order.
Without an intermediate durable state:
```text
calls: A, B, C
B finishes
C finishes
A is still running
process crashes
```
B and C exist only in process memory because A prevented source-ordered placement. Recovery treats them as unresolved and may replay or interrupt effects that already completed.
The solution separates two orders:
1. **outcome durability:** actual completion order;
2. **entry materialization:** assistant source order.
A complete finalized result becomes durable immediately in `pi.pending.entry`; the call becomes `outcome_ready`; placement happens later when every earlier source position is complete or ready.
## Goals
1. Never rerun a tool after its complete finalized outcome is durable.
2. Durably retain out-of-order parallel outcomes without violating transcript order.
3. Preserve the existing whole-tool `replay: "safe" | "never"` contract.
4. Support invocation-scoped durable memoization for Flue-style `step.do`.
5. Preserve tool-selected bounded progress checkpoints for reconnect and unsafe interruption recovery.
6. Fence late tool writes after outcome settlement, cancellation, or external finalization.
7. Keep final tree entries canonical, complete, bounded independently of progress snapshots, and immutable.
## Non-goals
- Exactly-once arbitrary external effects.
- A nested durable state machine for each `step.do` call.
- Inferring tool completion from progress output.
- Treating partial output as the canonical final result.
- Giving tools raw `Session`, `SessionMutator`, or bound storage-address access.
- Finalizing the public harness-native tool type in this document.
## Durable identities
Each call already reserves its result entry ID before execution. Use that as the stable public invocation identity:
```text
invocationId = resultEntryId
```
It is session-unique, survives safe replay, and is distinct from the provider's batch-local `toolCallId`, which a later assistant message may reuse.
The surrounding operation state continues to provide:
- `operationId`;
- `turnId`/generation step ID;
- `sourceIndex`;
- assistant entry ID;
- captured configuration and execution mode.
## Storage
### Existing bound values
```ts
operationToolArgs(operationId, turnId, sourceIndex)
// Effective validated arguments, persisted before effect admission.
pendingEntry(resultEntryId)
// Complete finalized ToolResultMessage while outcome_ready awaits placement.
```
### Invocation memos
Define the operation-owned address constructor in `session/values.ts`:
```ts
export const operationToolMemo = (
operationId: string,
invocationId: string,
memoName: string,
) => value<JsonValue>(
"pi.op.tool_memo",
`${operationId}:${invocationId}:${memoName}`,
);
```
`memoName` must be non-empty and contain no `:`. Names may use dots or slashes for application-local grouping. `setMemo(name, undefined)` deletes the exact bound value.
`scanValues(operationToolMemoPrefix(operationId))` permits defensive operation cleanup. `scanValues(operationToolMemoPrefix(operationId, invocationId))` permits atomic invocation cleanup when one outcome becomes ready. Core cleanup uses these owner-defined prefix constructors rather than repeating raw reserved namespace/key grammar.
### Partial tool output
The durable recovery value is the latest complete bounded progress snapshot selected by the tool. That is total current state, so use one bound value address:
```ts
export const pendingToolOutput = (
operationId: string,
invocationId: string,
) => value<AgentToolResult<unknown>>(
"pi.pending.tool_output",
`${operationId}:${invocationId}`,
);
```
This is auxiliary observation data. It never proves the effect succeeded or completed. The stored value has exactly the same content/details/usage shape as the live `partialResult`; recovery does not need a tool-specific progress codec.
The tool owns snapshot bounding, checkpoint cadence, and duplicate suppression. The harness owns synchronous enqueue, promise tracking, invocation fencing, and cleanup. The first API has no generic byte cap and does not truncate or reinterpret typed tool data; in-process tools are trusted to honor the bounded-snapshot contract. There is no tool-visible `flush()` method and no durable list of progress updates.
A crash may lose live updates newer than the latest committed checkpoint. JSONL physical growth is proportional to the size and frequency of distinct requested checkpoints until compaction; Memory and SQLite retain one current value. At 50 KiB every two seconds, JSONL's uncompacted worst case is approximately 15 MiB per ten minutes of continuously changing output.
## Tool update API
Keep the existing full-snapshot update callback and add harness-specific options:
```ts
export interface AgentHarnessToolUpdateOptions {
/** Request replacement of this invocation's durable recovery checkpoint. */
checkpoint?: true;
}
export type AgentHarnessToolUpdateCallback<TDetails> = (
partialResult: AgentToolResult<TDetails>,
options?: AgentHarnessToolUpdateOptions,
) => void;
```
`AgentHarnessTool` uses this callback instead of the legacy `AgentToolUpdateCallback`. The harness always supplies it, even without a live listener, because a tool may request persistence through it. The legacy `AgentTool` and old agent loop remain unchanged because they cannot honor durable checkpoints.
Every callback invocation remains an immediate live update. The callback is synchronous and returns `void`; `checkpoint: true` additionally requests persistence of that complete snapshot. It is not a durability acknowledgement. Internally, the harness retains the latest `events.emit(tool_update)` promise so existing listener delivery completes before `after_tool`; tools do not await or receive that promise.
Every `checkpoint:true` call synchronously enqueues one scalar replacement on the Session mutation line, attaches the ordinary harness-fault observer to that promise, and replaces the process-local `latestCheckpointWrite` reference. Writes themselves are neither dropped nor coalesced, and replacing the reference never leaves an earlier rejection unobserved:
- Session mutation FIFO preserves request order;
- each mutation verifies the same call remains `effect_pending`;
- completion of the latest promise implies completion of every earlier checkpoint write;
- tool-promise settlement stops accepting updates and awaits that latest promise before `after_tool`;
- a failed checkpoint commit faults the harness under the ordinary storage-fault rule.
A tool that requests checkpoints faster than storage can queue work in memory. Under the trusted-tool contract, cadence is the tool's responsibility. The built-in bash policy bounds this queue in ordinary use.
Tools should compare against their last requested checkpoint when duplicate suppression matters. Storage does not read and deep-compare the current scalar as part of every checkpoint.
### Bash policy
The built-in bash tool keeps its current 100 ms live update cadence. It requests a checkpoint at most once every two seconds and only when the complete bounded snapshot differs from its last requested checkpoint:
```ts
const BASH_UPDATE_THROTTLE_MS = 100;
const BASH_CHECKPOINT_INTERVAL_MS = 2_000;
```
The current `ShellCaptureProgress` already supplies a bounded snapshot: the last 2,000 lines or 50 KiB, plus truncation metadata and the overflow-file path. The initial empty update is live-only. Output volume never accelerates checkpoint frequency. A short tool may settle without writing any checkpoint because its complete final result commits instead.
## Tool-call state
Extend the call union with `outcome_ready`:
```ts
type ToolCall =
| {
status: "planned";
sourceIndex: number;
resultEntryId: string;
}
| {
status: "effect_pending";
sourceIndex: number;
resultEntryId: string;
replay: "never" | "safe";
}
| {
status: "outcome_ready";
sourceIndex: number;
resultEntryId: string;
terminate: boolean;
}
| {
status: "completed";
sourceIndex: number;
resultEntryId: string;
terminate: boolean;
};
```
`outcome_ready` means:
- execution, error normalization, and `after_tool` have finished, or the harness has constructed a final synthetic result;
- the complete final `ToolResultMessage` exists at `pendingEntry(resultEntryId)`;
- invocation memos and partial-output storage are gone;
- the tool must never execute again;
- the immutable result entry may not exist yet because an earlier call has not materialized.
The exact union may later carry small settlement metadata, but it must not duplicate the finalized result payload.
## State transitions
```text
planned
├─ real effect cleared → effect_pending
└─ immediate/synthetic → outcome_ready
effect_pending
├─ live effect settles → outcome_ready
├─ safe orphan replay settles→ outcome_ready
└─ unsafe orphan synthesis → outcome_ready
outcome_ready
└─ source position eligible → completed
```
An implementation may fuse `outcome_ready → completed` into the same transaction that finalizes an immediately placeable head call, but the semantic checks and tests must still cover durable `outcome_ready` for out-of-order calls. Prefer implementing the explicit two-transaction form first.
## Fresh execution
### Clearance and intent
Unchanged effect sandwich:
```text
planned
→ prepare arguments, run before_tool, validate replacements
→ TX[
set pi.op.tool_args,
set call = effect_pending(replay)
]
→ post-commit tool_start
→ admit tool execution
```
The invocation-scoped capability becomes active only for this durable `effect_pending` call.
### Partial output
Every `onUpdate(partialResult, options)` publishes the live update through the existing event/snapshot path. When `options.checkpoint === true`, the harness additionally requests a scalar replacement:
```text
TX[
setValue(pendingToolOutput(operationId, invocationId), partialResult)
]
```
The mutation verifies the same operation, turn, source position, and invocation remain `effect_pending`. It does not rewrite `pi.op.state`. A late checkpoint after settlement returns without committing.
The tool must checkpoint bounded complete snapshots, not growing unbounded values. Bash uses the same bounded `ShellCaptureProgress` snapshot it already sends live. Clients may render and locally retain live updates newer than the durable checkpoint, but those updates are explicitly process-local.
### Finalization to `outcome_ready`
When the tool promise settles:
1. synchronously stop accepting updates and expire the invocation capability;
2. await the latest tracked `tool_update` delivery and latest checkpoint-write promise; each implies completion of its preceding queue;
3. run `after_tool` when this is a real fresh or safely replayed result and cancellation did not prevent the hook;
4. construct the complete final `ToolResultMessage`;
5. commit the result as `outcome_ready`;
6. emit and await `tool_end` from the committed staging transition.
`setMemo()` returns a promise and tools must await it; `step.do` always does. An unawaited pre-return mutation is still enqueued before staging and is deleted by staging. Calls begun after capability expiry reject. No separate invocation-write drain exists.
Transaction:
```text
TX[
setValue(
pendingEntry(resultEntryId),
{ type: "message", payload: finalizedToolResultMessage },
),
deleteValue(pendingToolOutput(operationId, invocationId)),
deleteValue(memo.address) for every memo returned before commit by
scanValues(operationToolMemoPrefix(operationId, invocationId)),
setValue(operationState(operationId), call = outcome_ready(terminate))
]
```
The transaction is the linearization point after which the invocation can never replay. Its post-commit `tool_end` is therefore durable evidence that the finalized outcome is ready; it is no longer a pre-commit effect observation.
The staged message contains the final:
- text/image content;
- provider tool-call ID and tool name;
- details;
- `isError`;
- usage snapshot, when reported;
- added tool names;
- timestamp.
`terminate` remains orchestration state because it controls the batch continuation and is copied to the immutable entry's `terminate` field at placement. Staged `addedToolNames` do not affect the active tool set until that result materializes in the transcript.
## Source-ordered materialization
After any call becomes `outcome_ready`, find the contiguous ready prefix beginning at the first non-completed source position.
Example:
```text
[completed, outcome_ready, outcome_ready, effect_pending]
└──────── ready prefix ────────┘
```
Before the placement transaction, emit and await each finalized result's `message_start` and `message_end` in source order. Materialize the prefix in one transaction when practical, then emit `entry_added` and reported usage events in the same source order:
```text
TX[
insert result entry i from pendingEntry(i),
deleteValue(pendingEntry(i)),
insert tool usage row i if reported,
insert result entry i+1 with parent = result i,
deleteValue(pendingEntry(i+1)),
insert tool usage row i+1 if reported,
setValue(branchTip(lane), newest result),
setValue(operationState(operationId),
calls i..i+1 = completed and, when complete, next checkpoint)
]
```
Each inserted entry uses its already-reserved `resultEntryId`. Writes construct the parent chain in source order inside the transaction.
Tool-reported usage remains durable in the staged message until placement. The initial implementation writes its ledger row atomically with entry materialization, matching the current entry/usage ordering and avoiding a ledger row that references an entry not yet present. No usage ID reservation is needed because a failed placement transaction writes neither row nor completed state.
When the final call materializes, the same transaction calls `scanValues(operationToolArgsPrefix(operationId, turnId))` and deletes every returned address, and transitions to the correct checkpoint:
- every result terminates → `may_finish`, no final assistant required;
- otherwise → `need_assistant(false)`.
## Parallel execution
Outcome staging follows actual completion order. Entry materialization follows source order.
```text
A, B, C start
B finishes → B outcome_ready
C finishes → C outcome_ready
A finishes → A outcome_ready
materialize A, B, C
```
Crash after B and C stage:
```text
A effect_pending
B outcome_ready
C outcome_ready
```
Recovery applies unknown-outcome policy only to A. B and C require no tool registration or hook execution to become entries.
The durable invariant changes from “completed calls form a source-ordered prefix” to:
- completed calls form a source-ordered prefix;
- after that prefix, parallel calls may be `planned`, `effect_pending`, or `outcome_ready` in any mixture;
- only source-ordered materialization extends the completed prefix.
Sequential execution constructs at most one non-planned call after the completed prefix. Committed call state is trusted on restore; the owning procedure enforces this shape while creating and consuming transitions rather than through a broad restore audit.
## Unsafe recovery with partial output
For orphaned `effect_pending` with `replay: "never"`:
1. read `pendingToolOutput(operationId, invocationId)` when present;
2. preserve its bounded content and serializable details;
3. append a mandatory human-readable interruption marker;
4. construct a harness-owned `ToolResultMessage` with `isError: true`;
5. commit it as `outcome_ready` and clean invocation state.
The marker must state that output is partial and the external outcome is unknown. `isError: true` describes the result delivered to the model; it does not assert that the external effect failed.
Example final text suffix:
```text
[Tool execution was interrupted. The preceding output is the latest durable progress snapshot; newer live output may be missing, and the external outcome is unknown.]
```
Rules:
- do not run `after_tool` for this synthetic result;
- preserve checkpoint `usage` when present, but ignore checkpoint `addedToolNames` and `terminate` because progress never has final-result authority;
- set `terminate: false` and add no tools;
- an absent checkpoint value is also valid and yields only the interruption result;
- never infer completion from an apparent success line in partial output;
- cleanup of partial output and invocation memos is atomic with staging the synthetic result.
## Safe recovery
For orphaned `effect_pending` where both the stored and current declarations are `replay: "safe"`:
1. retain invocation memos;
2. atomically delete `pendingToolOutput(operationId, invocationId)`;
3. emit/reset process-local progress observation as needed;
4. rerun the tool with persisted arguments and the same `invocationId`;
5. completed `step.do` calls return their memoized values;
6. new partial output reconstructs a clean progress stream;
7. finalization follows the ordinary `outcome_ready` path.
Deleting old progress prevents duplicate chunks when replayed code emits progress again. A crash after the delete but before replay admission remains `effect_pending`; the next recovery repeats the same safe procedure.
If the current tool declaration is missing or no longer safe, use unsafe interruption recovery rather than suspending.
## Invocation memos
The harness-native tool call receives a purpose-built invocation capability conceptually equivalent to:
```ts
interface AgentHarnessToolInvocation {
readonly invocationId: string;
readonly operationId: string;
readonly turnId: string;
getMemo(key: string): Promise<JsonValue | undefined>;
setMemo(key: string, value: JsonValue | undefined): Promise<void>;
}
```
Each operation:
1. validates the memo name and checks process-local capability expiry;
2. synchronously enqueues work on the Session mutation line before returning its promise;
3. verifies the operation, turn, source position, and invocation are still the same `effect_pending` call when that job executes;
4. constructs and reads or writes only `operationToolMemo(operationId, invocationId, name)`;
5. rejects after capability expiry or durable ownership loss.
The durable check matters for authorized external finalization. In ordinary execution, a memo mutation initiated before the tool returns is FIFO-ordered before outcome staging; one initiated afterward fails the expired-capability check.
A late zombie callback can neither recreate memos after `outcome_ready` nor write into a later operation.
Memos are immediate durable replay state, not application-visible settlement state. They survive close/crash while the call remains `effect_pending` and are deleted when any real or synthetic outcome becomes ready.
Terminal cleanup defensively scans and deletes the operation-owned families:
```ts
scanValues(operationToolMemoPrefix(operationId))
scanValues(pendingToolOutputPrefix(operationId))
```
in addition to other operation-owned addresses. Each returned `StoredValue` supplies its exact bound address for `deleteValue`; no later operation receives a raw key.
## Flue-style `step.do`
Build `step.do` over invocation memos; it does not need its own harness state union:
```ts
interface ToolSteps {
do<T extends JsonValue>(
name: string,
effect: () => T | Promise<T>,
): Promise<T>;
}
```
Algorithm:
```text
validate deterministic unique name
→ getMemo("step/" + name)
→ present: return stored value
→ absent: run effect
→ setMemo("step/" + name, value)
→ await durability
→ return value
```
Crash behavior:
```text
before/during effect → effect may run on replay
effect returned, memo not committed → effect may run on replay
memo committed → replay returns memo
step A memoized, step B interrupted → rerun tool; A skips, B runs
```
This is exactly-once recorded and at-least-once executed. It does not make arbitrary external effects exactly once. An application may derive a stable external idempotency key from `(invocationId, stepName)` when the external API supports one.
Errors are not memoized. A thrown effect either contributes to the current tool result or runs again after safe whole-tool recovery.
Do not add per-step `replay: "never"` in this slice. Supporting it correctly requires a nested `planned → effect_pending → completed` state and an explicit unknown-outcome policy. Whole-tool replay policy is sufficient for the Flue use case already discussed.
Within one live execution, calling the same step name twice is an invariant error. Names must be deterministic across safe replay.
## Application persistent state
Flue-style application state is distinct from invocation memos.
Invocation memo:
```text
step completed → memo becomes visible immediately
```
Application state:
```text
tool stages state change
→ crash before outcome_ready: state must not appear committed
→ outcome_ready: state and finalized result become visible together
```
Do not implement application state by calling `Session.setValue()` directly during execution.
Two valid implementation stages:
1. Flue keeps its state externally and atomically stores application state plus a full result memo keyed by `invocationId`.
2. A later harness API accepts staged application value writes and promotes them in the `outcome_ready` transaction.
The second option is required only if Flue's `usePersistentState` moves into harness-owned session values. Its public typing and conflict semantics remain an open design item for the harness-native tool discussion.
## Cancellation
Cancellation reconciliation never replays a restored tool.
- `planned` calls receive a synthetic aborted result and become `outcome_ready`;
- a live started call may finalize its real local result under cancelled control, then become `outcome_ready` with `terminate: false`;
- restored `effect_pending` calls use an interrupted synthetic result, optionally including partial output, regardless of safe replay declaration;
- existing `outcome_ready` calls are preserved and materialized in source order;
- invocation memos and partial output are deleted with each staged cancellation outcome;
- no `before_tool` or `after_tool` starts during restored synthetic reconciliation.
The aborted terminal transaction runs only after every call outcome has materialized and accepted deferred writes have drained under the existing cancellation rules.
## Close and external finalization
Close is still a controlled crash:
- memo/checkpoint mutations already enqueued under the admission barrier may finish;
- live output newer than the latest committed tool-requested checkpoint may be lost;
- no synthetic outcome or cancellation marker is written;
- durable state remains at `effect_pending` or `outcome_ready`.
External finalization deletes operation-owned arguments, invocation memos, partial output, staged pending outcomes, and other pending entries in its terminal transaction. A live task that later tries to stage an outcome fails the ownership fence and stops through `OperationEnded`.
## Restore and consumption-time reads
Base restore constructs the trusted lane/operation projection from required owner values. It does not hydrate or semantically audit tool arguments, invocation memos, progress checkpoints, staged outcomes, completed entries, completed-prefix shape, or captured execution-mode relationships.
The procedure responsible for the current typed state performs only its exact consumption-time reads:
### `planned`
Clearance needs no auxiliary restore read. It prepares the call and writes arguments before effect admission.
### `effect_pending`
Activation reads `operationToolArgs(operationId, turnId, sourceIndex)` and optionally reads `pendingToolOutput(operationId, invocationId)`. Missing required arguments are an invariant defect at consumption. Invocation memos are read only through the scoped capability. Safe replay, unsafe interruption, and snapshots use no broad prefix scan.
### `outcome_ready`
Materialization reads `pendingEntry(resultEntryId)`. Its absence or wrong trusted message relationship is an invariant defect when materialization consumes it. No tool identity or effect recovery is needed.
### `completed`
Ordinary dispatch performs no restore-time entry audit. Context/tree reads later consume the immutable entry through their normal typed paths.
Every live mutation still verifies current operation, turn, source position, invocation, and status on the Session line. Those checks fence concurrent settlement, cancellation, and external finalization; they are not historical restore validation. Terminal prefix cleanup remains defensive and does not make orphan scans part of restore.
## Snapshots and reconnect
A reconnecting client may see:
- live process-local progress newer than the latest durable checkpoint before disconnect;
- after process replacement, only the latest committed bounded checkpoint;
- `outcome_ready` calls as settled rows in `runningTools` until source-ordered materialization;
- completed calls in the transcript.
`LaneSnapshot.operation.runningTools` is a discriminated union. An effect-pending tool has `status: "running"` and an optional `result` containing the latest complete progress snapshot, falling back to the durable checkpoint after reopen. An outcome-ready call has `status: "settled"`, its required complete final `result`, and `isError`; it remains there until its immutable result entry's `entry_added` removes the row and places the same presentation in the transcript. Planned and completed calls are omitted.
## Events and hooks
- `tool_start` begins public processing presentation for a fresh call; it is emitted from the commit that establishes effect intent or a synthetic staged outcome and does not by itself prove an external effect started. It carries effective arguments for an intended effect and source arguments for an immediate synthetic result.
- live progress events and durable progress checkpoints do not prove completion.
- The harness awaits the latest `tool_update` delivery before `after_tool`, preserving the existing listener ordering without making `onUpdate` async.
- `tool_end` carries the complete finalized result after its `outcome_ready` staging commit, in completion order. It is durable settlement evidence and does not repeat the arguments from `tool_start`.
- for a fresh blocked, invalid, truncated, or planned-cancellation synthetic outcome, the staging commit emits `tool_start` followed by `tool_end`; these paths still run no tool effect or post-effect hook. Cancellation after effect intent uses the earlier intent-bound start and a staging-bound end.
- an unsafe restored effect is already represented as running by the initial snapshot and may emit only a recovery-tagged `tool_end` when interruption synthesis stages.
- message lifecycle and `entry_added` occur when the staged result materializes, not when it first becomes `outcome_ready`; `entry_added` removes only that settled row.
- passive listeners cannot mutate invocation state reentrantly.
Instrumented-storage tests assert `intent commit → tool_start → tool_update* → outcome staging → tool_end → source-ordered placement` for execution and `outcome staging → tool_start → tool_end → source-ordered placement` for fresh synthetic results. Historical events are not replayed; a safely replayed execution emits recovery `tool_start` from its checkpoint-clear commit and `tool_end` from outcome staging.
## Races
| Race | Required result |
|---|---|
| checkpoint vs tool settlement | every accepted checkpoint was enqueued first; settlement awaits the latest promise, then staging deletes the checkpoint value; a late update is ignored |
| memo write vs `outcome_ready` | an awaited or pre-return-enqueued write precedes staging and is then deleted; a post-return call rejects; external finalization first causes the durable ownership check to reject |
| B outcome vs earlier A settlement | B stages independently; placement waits for A |
| crash after outcome staging | tool never replays; pending result later materializes |
| crash during source-prefix placement | transaction exposes either none or all of that placement prefix |
| safe replay vs old partial output | the old bound checkpoint value is deleted before replay emits new progress |
| cancellation vs real settlement | Session mutation order chooses real cancelled-control result or synthetic reconciliation; at most one outcome stages |
| terminal finalization vs late result | terminal ownership wins or outcome stages first; late task never recreates operation data |
| external finalization vs memo/checkpoint mutation | mutation first is removed by terminal cleanup; finalization first makes the mutation's durable ownership check reject |
## Invariants
1. `invocationId` equals the reserved result entry ID and is stable across safe replay.
2. A call in `outcome_ready` or `completed` never executes again.
3. Every `outcome_ready` call has exactly one complete matching `pi.pending.entry` value.
4. Completed calls form a source-ordered prefix.
5. Only source-ordered materialization extends that prefix.
6. Parallel calls after the completed prefix may mix planned, effect-pending, and outcome-ready states.
7. Invocation memos exist only while their call is `effect_pending`.
8. Partial output is auxiliary and never establishes effect completion.
9. Unsafe synthetic results explicitly state that captured output is incomplete and the external outcome is unknown.
10. Staging an outcome atomically deletes its invocation memos and partial output.
11. Materialization atomically inserts the immutable entry and deletes its staged pending value.
12. A late invocation capability cannot write after outcome settlement or operation loss.
13. `step.do` values are memoized only after their memo write commits; effects remain at-least-once.
14. Operation terminal cleanup leaves no tool args, invocation memos, partial output, or staged outcomes.
## Required tests
### State and restore
- every trusted planned/effect-pending/outcome-ready/completed projection restores without auxiliary reads;
- base restore does not audit completed-prefix or execution-mode relationships;
- effect-pending consumption reads exact required arguments and optional bounded checkpoint;
- outcome-ready consumption reads the exact staged result;
- missing required arguments or staged result fails at the consuming procedure, not base restore;
- invocation memos and partial output are absent after staging;
- snapshot/activation hydrate only the exact bounded checkpoint value when needed.
### Parallel ordering
- B and C stage while A remains pending;
- crash/reopen proves B and C never replay;
- A recovery followed by one source-ordered A/B/C placement transaction;
- mixed `[completed, outcome_ready, planned, effect_pending, outcome_ready]` state;
- immediate synthetic outcomes stage out of order;
- all-terminating batch transitions correctly after ordered placement.
### Replay and interruption
- safe replay uses persisted arguments and the same invocation ID;
- safe replay preserves step memos but clears old partial output;
- current declaration downgrade from safe to never interrupts;
- unsafe recovery with no checkpoint and with a complete bounded checkpoint;
- synthetic result is error/incomplete/unknown and never runs `after_tool`;
- cancellation never safely replays a restored call.
### Invocation memos and `step.do`
- set/get/delete memo with invocation-address scoping;
- completed step skips effect after reopen;
- crash before memo commit reruns effect;
- crash after memo commit returns memo;
- several completed steps followed by one interrupted step;
- duplicate live step names reject;
- memo write racing outcome staging;
- capability write after expiry, cancellation outcome staging, and external finalization rejects;
- terminal cleanup removes crash-leaked memos.
### Partial output
- ordinary updates stay live-only;
- `checkpoint: true` writes the complete bounded snapshot;
- tool-selected checkpoint cadence and duplicate suppression;
- bash emits live snapshots at 100 ms and requests distinct checkpoints at most every two seconds;
- every selected checkpoint enqueues one write, and awaiting the latest promise at tool settlement implies all earlier writes completed;
- a checkpoint after outcome staging cannot recreate the address's value;
- Memory, JSONL, and SQLite restore the same checkpoint value;
- JSONL growth follows checkpoint cadence rather than raw bash output volume;
- terminal compaction reclaims superseded/deleted checkpoint snapshots according to the existing dead-byte policy.
### Atomicity and instrumentation
- exact intent, synchronous update acceptance, asynchronous update-delivery, `after_tool`, outcome-ready staging, post-commit `tool_end`, source-ordered message lifecycle, and materialization order;
- outcome staging is atomic with memo/output cleanup;
- materialization is atomic with pending deletion, usage, tip, and state;
- crash at every boundary;
- no effect starts before intent;
- no effect or hook starts from `outcome_ready`;
- terminal transaction removes every operation-owned tool value.
## Implementation map
Expected runtime areas:
- operation-state types in `packages/agent/src/harness/session/types.ts`;
- trusted restore projection and exact consumption-time address reads;
- tool-batch procedure and source-ordered materialization;
- terminal cleanup and cancellation reconciliation;
- harness-specific update options, snapshots, and events;
- invocation-scoped capability implementation on the Session mutation line;
- backend conformance through the bound value/list APIs;
- instrumented-storage transaction assertions.
Concrete built-in address constructors in `session/values.ts`:
```text
operationToolMemo(operationId, invocationId, name) → value("pi.op.tool_memo", ...)
pendingToolOutput(operationId, invocationId) → value("pi.pending.tool_output", ...)
pendingEntry(resultEntryId) → value("pi.pending.entry", ...)
```
Implement `outcome_ready` and invocation memos before progress checkpoints. The state solves incorrect parallel replay by itself; checkpoints improve reconnect observation and unsafe interruption diagnostics without becoming completion authority.

735
third_party/pi/packages/agent/docs/values.md vendored
View File

@@ -1,735 +0,0 @@
# Typed values and lists
This document specifies the mutable storage primitive used by Session, the harness, and applications.
The public abstraction is a **bound typed address**:
- `value<T>(namespace, key?)` names one replaceable durable value;
- `list<T>(namespace, key?)` names one append-only durable list whose elements have type `T`.
The namespace/key pair is bound once when the address is constructed. Every later operation receives only that address. Application code therefore writes:
```ts
const state = value<ApplicationState>("my-app.state");
const events = list<ApplicationEvent>("my-app.events");
await session.getValue(state, context);
await session.setValue(state, nextState, context);
await session.readList(events, { limit: 100 }, context);
await session.appendList(events, event, context);
```
It does **not** repeatedly pass a second unexplained key:
```ts
// Not the API.
await session.readList(events, "another-key", { limit: 100 }, context);
```
When an application genuinely has keyed instances, it constructs the address for that instance:
```ts
const workspaceEvents = (workspaceId: string) =>
list<ApplicationEvent>("my-app.events", workspaceId);
await session.readList(workspaceEvents("pi"), { limit: 100 }, context);
```
Storage may physically index the address as `(kind, namespace, key)`, but that representation does not leak into each read or write call. Storage, Session, harness code, and applications use the same address vocabulary. There is no global value-type map, dynamic registry, token catalog, or separate application-state storage mechanism.
## Goals
1. Give one exact durable address one compile-time value type.
2. Let applications define scalar values and lists without declaration merging or editing a core type map.
3. Use the same typed addresses and operation names from Storage through Session.
4. Preserve current scalar replacement semantics.
5. Append one list element without reading or rewriting existing elements.
6. Give every list element its own existing session-global transaction `seq`, used for ordering and pagination.
7. Commit values and list elements atomically with entries and usage.
8. Produce identical logical behavior on Memory, JSONL, and SQLite.
9. Keep list reads bounded and explicit.
10. Keep scalar operation state authoritative; auxiliary lists never select a recovery state.
## Non-goals
This slice does not define:
- assistant-frame contents or reduction semantics;
- tool-progress semantics;
- runtime validation of trusted in-process values;
- per-element or per-list byte limits;
- list truncation or per-element deletion;
- a generic event log, journal, stream-resumption protocol, or operation reducer;
- globally registering address objects;
- exposing raw Session or transaction access to tools.
Consumers own address construction, content limits, cleanup points, fork policy, migration policy, and consumption-time hydration. `assistant-durability.md` defines the first list consumer.
## Bound address model
```ts
declare const storedValueType: unique symbol;
interface StoredAddressBase {
/** Stable persisted grouping name. */
readonly namespace: string;
/** Exact member inside that grouping. Empty is legal. */
readonly key: string;
readonly kind: "value" | "list";
}
export interface Value<T> extends StoredAddressBase {
readonly kind: "value";
/** Compile-time only and invariant in T. */
readonly [storedValueType]?: (value: T) => T;
}
export interface ValueList<T> extends StoredAddressBase {
readonly kind: "list";
/** T is one element, not the whole list. */
readonly [storedValueType]?: (value: T) => T;
}
export function value<T>(namespace: string, key = ""): Value<T> {
validateAddress(namespace, key);
return Object.freeze({ namespace, key, kind: "value" });
}
export function list<T>(namespace: string, key = ""): ValueList<T> {
validateAddress(namespace, key);
return Object.freeze({ namespace, key, kind: "list" });
}
```
The phantom function makes `T` invariant: an address for one type cannot silently widen to another. It has no runtime field.
Rules:
- `namespace` must be non-empty;
- namespace `pi` and every `pi.*` namespace are reserved for built-ins by contract;
- applications that construct a reserved address are defective trusted in-process code; no runtime privilege split, registry, or catalog exists;
- neither component may contain the Memory backend's internal separator (`\u0000`);
- an empty key is valid and is the natural address for one application-wide value or list;
- object identity has no durable meaning;
- separately constructed addresses with the same `(kind, namespace, key)` identify the same durable location;
- constructing the same durable location with incompatible TypeScript types is a trusted-programming defect;
- scalar and list addresses may not share the same `(namespace, key)` in one storage version; violating this is a trusted-programming defect and storage performs no cross-kind collision check;
- changing an address's namespace, key, kind, or incompatible value shape requires migration.
The two components remain separate rather than concatenated. Dynamic application keys and operation IDs therefore require no escaping convention beyond the storage separator rule.
### Exact addresses, not families
An address names one value or one list. Internal code uses small constructors when it has dynamic keys:
```ts
export const branchTip = (lane: string) =>
value<string | null>("pi.branch.tip", lane);
export const operationState = (operationId: string) =>
value<OperationState>("pi.op.state", operationId);
export const operationToolArgs = (
operationId: string,
stepId: string,
sourceIndex: number,
) => value<Record<string, JsonValue>>(
"pi.op.tool_args",
`${operationId}:${stepId}:${sourceIndex}`,
);
export const pendingAssistantFrames = (
operationId: string,
responseEntryId: string,
) => list<AssistantMessageFrame>(
"pi.pending.assistant_frame",
`${operationId}:${responseEntryId}`,
);
```
This encapsulates each key grammar at its owner. Call sites receive an already-bound typed address:
```ts
await reader.getValue(operationState(operationId));
await reader.readList(pendingAssistantFrames(operationId, responseEntryId), options);
```
### No global value map
Delete the existing global namespace-to-type maps:
```ts
interface RegisterValues { /* delete */ }
interface ListRegisterValues { /* delete */ }
type RegisterNamespace = keyof RegisterValues; // delete
```
A type belongs to an address constructor instead:
```ts
export const applicationState = value<MyApplicationState>("my-app.state");
export const applicationEvents = list<MyApplicationEvent>("my-app.events");
```
Applications should use a stable, collision-resistant namespace prefix. Namespace `pi` and the complete `pi.*` prefix are reserved for built-ins by contract; similar-looking names such as `pi2` remain legal. The same `value()` and `list()` constructors serve core and application code. Tests assert that every built-in address uses its reserved prefix. There is no runtime privilege split, registry, or catalog.
## Built-in addresses
Built-in constructors live together in `packages/agent/src/harness/session/values.ts` and are imported directly by consumers. Representative definitions:
```ts
export const branchTip = (lane: string) =>
value<string | null>("pi.branch.tip", lane);
export const laneConfig = (lane: string) =>
value<LaneConfiguration>("pi.lane.config", lane);
export const laneState = (lane: string) =>
value<LaneState>("pi.lane.state", lane);
export const operationResult = (operationId: string) =>
value<OperationResultRecord>("pi.result", operationId);
/** Used only by scanValues() to enumerate Branch names. */
export const branchTipInventoryPrefix = () =>
value<string | null>("pi.branch.tip");
export const operationMeta = (operationId: string) =>
value<OperationMeta>("pi.op.meta", operationId);
export const operationState = (operationId: string) =>
value<OperationState>("pi.op.state", operationId);
export const operationToolArgs = (operationId: string, stepId: string, sourceIndex: number) =>
value<Record<string, JsonValue>>(
"pi.op.tool_args",
`${operationId}:${stepId}:${sourceIndex}`,
);
export const operationToolMemo = (operationId: string, invocationId: string, name: string) =>
value<JsonValue>("pi.op.tool_memo", `${operationId}:${invocationId}:${name}`);
export const operationPreparation = (operationId: string, taskId: string) =>
value<DurableStructuralPreparation>(
"pi.op.preparation",
`${operationId}:${taskId}`,
);
/** Prefix addresses are exported only for namespace-scoped scanValues(). */
export const operationToolArgsPrefix = (operationId: string, stepId?: string) =>
value<Record<string, JsonValue>>(
"pi.op.tool_args",
stepId === undefined ? `${operationId}:` : `${operationId}:${stepId}:`,
);
export const operationToolMemoPrefix = (operationId: string, invocationId?: string) =>
value<JsonValue>(
"pi.op.tool_memo",
invocationId === undefined ? `${operationId}:` : `${operationId}:${invocationId}:`,
);
export const operationPreparationPrefix = (operationId: string) =>
value<DurableStructuralPreparation>("pi.op.preparation", `${operationId}:`);
export const pendingEntry = (entryId: string) =>
value<PendingEntry>("pi.pending.entry", entryId);
export const pendingToolOutput = (operationId: string, invocationId: string) =>
value<AgentToolResult<unknown>>(
"pi.pending.tool_output",
`${operationId}:${invocationId}`,
);
export const pendingAssistantFrames = (operationId: string, responseEntryId: string) =>
list<AssistantMessageFrame>(
"pi.pending.assistant_frame",
`${operationId}:${responseEntryId}`,
);
export const pendingToolOutputPrefix = (operationId: string) =>
value<AgentToolResult<unknown>>("pi.pending.tool_output", `${operationId}:`);
export const sessionName = value<string>("pi.session.name");
export const entryLabel = (entryId: string) => value<string>("pi.entry.label", entryId);
```
`OperationMeta` is immutable acceptance metadata stored at `pi.op.meta`. The process-local `Operation` projection is `{ meta: OperationMeta, state: OperationState }`, assembled from the separate metadata and state values; it is never stored at one address.
The five exported scan-prefix constructors are `branchTipInventoryPrefix`, `operationToolArgsPrefix`, `operationToolMemoPrefix`, `operationPreparationPrefix`, and `pendingToolOutputPrefix`. Their addresses are consumed only by `scanValues()`.
Applications define their own `value()` and `list()` addresses directly; there is no built-in custom application-state namespace or custom-state API. `AgentHarnessToolInvocation.getMemo()` and `setMemo()` are invocation-fenced capabilities over `operationToolMemo(...)`, not raw Session access. Invocation memos remain operation-owned and are deleted when their tool outcome becomes durable.
Tests assert that built-in constructors produce the documented kind, namespace, and key grammar. Because constructors may be dynamic, there is no runtime catalog that tries to enumerate every possible address.
## Shared read API
Storage, Session, SessionReader, and SessionMutator use the same read signatures:
```ts
export interface StoredValue<T> {
address: Value<T>;
value: T;
seq: number;
}
export interface ListElement<T> {
/** Global transaction-write sequence assigned by storage. */
seq: number;
value: T;
}
export interface ListCursor {
seq: number;
}
export interface ListReadOptions {
/** Exclusive cursor. */
cursor?: ListCursor;
/** Default: asc. */
order?: "asc" | "desc";
/** Query-page size. Default: 1,000. Values above 10,000 clamp to 10,000. */
limit?: number;
}
interface ValueReader {
getValue<T>(address: Value<T>): Promise<StoredValue<T> | undefined>;
/** Internal bounded-prefix operation. The address key is interpreted as a prefix. */
scanValues<T>(prefix: Value<T>): Promise<StoredValue<T>[]>;
readList<T>(
address: ValueList<T>,
options?: ListReadOptions,
): Promise<ListElement<T>[]>;
}
```
`scanValues(prefixAddress)` scans scalar addresses with exactly that namespace and keys beginning with the bound key, returning them in key-ascending order. Core call sites use only the exported prefix constructors above so raw namespace/key grammar stays in `session/values.ts`. Prefix addresses are passed only to `scanValues()`, never to exact get/set/delete operations. There is no unrestricted cross-namespace dump. Ordinary application reads use exact addresses.
`Session` exposes direct one-transition writes using the same addresses:
```ts
interface Session extends ValueReader {
setValue<T>(
address: Value<T>,
next: NoInfer<T>,
context: Context,
): Promise<void>;
deleteValue<T>(address: Value<T>, context: Context): Promise<void>;
appendList<T>(
address: ValueList<T>,
element: NoInfer<T>,
context: Context,
): Promise<void>;
deleteList<T>(address: ValueList<T>, context: Context): Promise<void>;
}
```
Purpose-specific helpers such as `getName()`, `setName()`, `getLabel()`, and `setLabel()` may remain thin wrappers over built-in addresses. Applications define and use their own scalar/list addresses directly.
`SessionMutator` remains a read capability plus one atomic `commit(writes)`. It does not expose direct `setValue()`/`appendList()` methods that would consume its only commit separately; callers construct a typed write array and commit it together.
## Typed transaction writes
Writes are constructed through typed helpers. Entry and usage constructors hide their storage discriminants; value/list erasure happens only after the helper has checked the address/value type relationship:
```ts
interface EntryWrite {
kind: "entry";
entry: NewEntry;
}
interface UsageWrite {
kind: "usage";
row: Omit<UsageRow, "seq">;
}
interface ValueSetWrite {
kind: "value";
op: "set";
namespace: string;
key: string;
value: unknown;
}
interface ValueDeleteWrite {
kind: "value";
op: "delete";
namespace: string;
key: string;
}
interface ListAppendWrite {
kind: "list";
op: "append";
namespace: string;
key: string;
value: unknown;
}
interface ListDeleteWrite {
kind: "list";
op: "delete";
namespace: string;
key: string;
}
export function insertEntry(entry: NewEntry): EntryWrite;
export function insertUsage(row: Omit<UsageRow, "seq">): UsageWrite;
export function setValue<T>(address: Value<T>, next: NoInfer<T>): ValueSetWrite;
export function deleteValue<T>(address: Value<T>): ValueDeleteWrite;
export function appendList<T>(address: ValueList<T>, element: NoInfer<T>): ListAppendWrite;
export function deleteList<T>(address: ValueList<T>): ListDeleteWrite;
```
`NoInfer<T>` makes the address authoritative. TypeScript must not infer a wider `T` from an incompatible write value.
`Write` includes all six helper return types. One transaction may mix every write kind atomically. Harness and application code use the helpers rather than manually constructing storage write shapes.
The direct Session methods and transaction helpers intentionally use the same operation names. One performs and commits a single Session mutation; the other constructs a write for an explicitly composed transaction.
## Scalar semantics
For one `Value<T>` address:
- `setValue` replaces the current value;
- `deleteValue` removes it;
- deleting an absent value is a no-op;
- set after delete recreates it;
- there is no retained value history;
- the current value records the `seq` of its latest set;
- a failed transaction exposes neither the scalar write nor any sibling write.
## List semantics
One append write carries one immutable element. A transaction appending several elements contains several append writes. Every write receives its existing globally increasing transaction sequence:
```text
TX[
appendList(frames, A), // seq 41
setValue(operationState, X), // seq 42
appendList(frames, B), // seq 43
]
```
Reading `frames` returns `A`, then `B`. Gaps from unrelated writes are expected. A list element's `seq` is session-global and unique to that committed write; it is an ordering/cursor identity, not an application domain ID. Applications that need domain identity include it in `T`.
Rules:
- append never reads existing elements;
- an element is immutable after commit;
- `deleteList(address)` removes every element at that exact address;
- deleting an absent list is a no-op;
- delete followed by append in one transaction creates a fresh list atomically;
- there is no per-element update, delete, insertion, or truncation;
- all validation and serialization required to admit a transaction completes before Memory state changes;
- a failed transaction exposes none of its list or non-list writes.
“Append-only” describes elements while the list exists. Whole-list deletion is lifecycle cleanup, not element mutation.
### List reads
- ascending reads return `seq > cursor.seq`;
- descending reads return `seq < cursor.seq`;
- results are ordered according to `order` before `limit` is applied;
- absent and empty both return `[]`;
- callers continue with the last returned element's `seq`;
- an empty page ends iteration;
- `limit` is only the query-page size: it must be a positive safe integer, defaults to 1,000, and values above 10,000 clamp to 10,000; it never limits total list length or bytes.
```ts
let cursor: ListCursor | undefined;
while (true) {
const page = await reader.readList(events, { cursor, order: "asc", limit: 100 });
if (page.length === 0) break;
consume(page);
cursor = { seq: page[page.length - 1]!.seq };
}
```
A cursor is a sequence filter, not a snapshot or list-incarnation token. Concurrent later appends may appear on later ascending pages. Whole-list deletion may make a cursor stale; reads simply apply its sequence comparison to currently surviving elements.
Do not add an unbounded “read the whole list” helper.
## Assistant partial frames
Assistant partial durability is the first built-in list consumer:
```ts
const frames = pendingAssistantFrames(operationId, responseEntryId);
```
`AssistantMessageFrame`, `AssistantMessageFrameEncoder`, and `reduceAssistantMessageFrames()` come from `@earendil-works/pi-ai`. Do not define a second frame codec or reducer.
For every convertible non-terminal provider event, the assistant procedure:
```text
convert event to frame
→ synchronously enqueue appendList(frames, frame) on the Session mutation line
→ attach the ordinary harness-fault observer to that returned promise
→ replace the process-local latestFrameWrite reference
→ emit and await the existing message event
→ consume the next provider event
```
The provider loop does not await storage for every frame. Synchronous enqueue preserves provider-event order. Replacing the latest-promise reference never leaves an earlier rejection unobserved because every promise receives the fault observer. Bounded output bounds queued work. On stream settlement, the procedure stops frame admission and awaits the latest append promise before `after_response`; Session mutation FIFO means that completion implies every earlier append completed. There is no timer, batcher, coalescer, or flush API.
Scalar assistant `effect_pending` remains authoritative. Each append verifies that the same operation, attempt, and response ID still own the lane when its mutation executes. Frames never prove request admission, completion, success, or failure.
Final or synthetic assistant settlement deletes the exact list atomically with its immutable response, usage, Branch tip, and next scalar state:
```text
TX[
insert final assistant entry,
insert usage,
deleteList(frames),
setValue(operationState(operationId), nextState),
]
```
`assistant-durability.md` defines frame conversion, unknown-outcome synthesis, cancellation, deferred polling, snapshots, and event ordering.
## Restore policy
Scalar operation state remains the sole restart authority:
1. construct the trusted lane/operation projection from required scalar values;
2. trust committed typed values rather than auditing every referenced payload or phase relationship;
3. when a procedure or snapshot consumes auxiliary state, derive its exact bound address from current typed scalar state;
4. hydrate only the bounded scalar values or list pages that consumer requires.
A missing auxiliary list is legal unless its consumer explicitly requires an element. List contents never prove that an external effect completed. Live mutations still verify current operation, phase, attempt, and reserved identity as concurrency fencing; that is not restore validation.
Base restore does not enumerate lists. For assistant frames, snapshot or recovery derives `pendingAssistantFrames(operationId, responseEntryId)` only while consuming a typed assistant/deferred `effect_pending` state.
Each list consumer defines:
- address grammar;
- element and total-byte bounds;
- page/hydration budget;
- cleanup transitions;
- fork and migration policy.
## Memory backend
Memory may keep separate maps for current values and list elements:
```ts
const scalarValues = new Map<string, StoredValue<unknown>>();
const listValues = new Map<string, ListElement<unknown>[]>();
function physicalKey(address: StoredAddressBase): string {
return `${address.namespace}\u0000${address.key}`;
}
```
- scalar set replaces one map value;
- scalar delete removes it;
- list append pushes the already-sequenced element;
- list delete removes the complete array;
- list read filters by exclusive cursor and slices to the validated limit;
- transaction preparation completes before entries, values, lists, usage, or stats mutate.
Storage snapshots used by JSONL/fork tooling include current scalar values and surviving list elements with original sequence numbers.
## SQLite backend
The logical schema has one current-value table and one list-element table:
```sql
CREATE TABLE scalar_values (
namespace TEXT NOT NULL,
key TEXT NOT NULL,
seq INTEGER NOT NULL,
value TEXT NOT NULL,
PRIMARY KEY (namespace, key)
) WITHOUT ROWID;
CREATE TABLE list_values (
namespace TEXT NOT NULL,
key TEXT NOT NULL,
seq INTEGER NOT NULL,
value TEXT NOT NULL,
PRIMARY KEY (namespace, key, seq)
) WITHOUT ROWID;
```
WP01 replaces the unfinished format-4 schema in place: edit `sqlite/migrations/001_initial.sql`, rename the physical `registers` table to `scalar_values`, add `list_values`, and keep `SQLITE_STORAGE_VERSION = 1`. There is no migration runner in this WIP implementation, and pre-WP01 SQLite files are unsupported. Do not add migration machinery in this package.
List operations:
```sql
INSERT INTO list_values(namespace, key, seq, value) VALUES (?, ?, ?, ?);
SELECT seq, value FROM list_values
WHERE namespace = ? AND key = ? AND seq > ?
ORDER BY seq ASC LIMIT ?;
SELECT seq, value FROM list_values
WHERE namespace = ? AND key = ? AND seq < ?
ORDER BY seq DESC LIMIT ?;
DELETE FROM list_values WHERE namespace = ? AND key = ?;
```
For a missing cursor, omit the sequence predicate. Every write participates in the existing `BEGIN IMMEDIATE` transaction; writable Session ownership belongs to the host lifecycle, not SQLite storage. Assert with `EXPLAIN QUERY PLAN` that paging uses the primary key and no temporary sort.
## JSONL backend
Logical records carry the bound address's physical components:
```jsonl
{"kind":"list","op":"append","seq":41,"namespace":"pi.pending.assistant_frame","key":"O:R","value":{"type":"text_delta","contentIndex":0,"delta":"hi"}}
{"kind":"list","op":"delete","seq":52,"namespace":"pi.pending.assistant_frame","key":"O:R"}
```
Scalar records use `kind:"value"` with `op:"set"|"delete"`. WP01 keeps JSONL format 4 and storage version 1 but replaces the unfinished record spelling in place; pre-WP01 format-4 files are unsupported and no legacy `kind:"register"` decoder remains.
Replay folds records into the Memory state:
- scalar set replaces the current address;
- scalar delete removes it;
- list append adds `{ seq, value }`;
- list delete removes the complete list.
A transaction remains one physical JSONL line, using an array for multiple writes. Torn-tail handling therefore remains atomic without new framing.
### Snapshot compaction
Compaction writes every surviving list element with its original `seq`, merged in sequence order with surviving entries, scalar values, and usage rows. Do not collapse one live list into a synthetic element or assign new sequence numbers; either change breaks cursors and backend equivalence.
Deleted lists produce no snapshot records. Snapshot rewrites persist `nextSeq` in the format-4 header so dropping the latest delete cannot permit sequence reuse; ordinary append-only files may omit that field and derive it from replayed writes.
## Forks and rewrites
Fork and precise-rewrite code decides policy per concrete address grammar:
- operation-owned `pi.op.*` scalar values are not copied into an idle fork;
- immutable `pi.result` operation records are not copied by forks;
- `pi.pending.entry`, `pi.pending.tool_output`, and `pi.pending.assistant_frame` values/lists are not copied;
- lane and semantic session values follow their existing scope rules;
- application-defined values/lists are not copied by the generic fork; a consuming feature must add an explicit address-specific policy before relying on copied application state.
A precise rewrite retaining list elements preserves their `seq` values unless it explicitly remaps the entire destination sequence space.
## Schema evolution
A bound address's namespace, key grammar, kind, and value type are durable schema:
- changing namespace or key grammar requires explicit address migration;
- changing scalar to list or list to scalar requires explicit migration;
- storage never infers or coerces kind from observed records;
- changing TypeScript value shape requires total value migration when old stored values are incompatible;
- a list migration pages elements in sequence order and either maps them while preserving `seq` or deletes the complete list;
- a migration must not load an unbounded logical list at once.
Adding generic list storage replaces the current WIP backend schema in place. Constructing a new application address with no persisted value requires no migration.
## Instrumentation and telemetry
The instrumented storage decorator exposes the address-based read API and records committed erased writes in exact transaction order.
Telemetry session-write item kinds distinguish scalar-value writes from list writes. Namespace/key names may be attributes when the telemetry schema permits them, but values, assistant frames, prompts, and tool output never enter telemetry.
Append-path tests prove that no `readList` call occurs before append commit. Frame-persistence promises always receive the harness fault observer, even when an earlier promise is no longer the latest settlement-order reference.
## Invariants
1. One bound address has one stable namespace/key/kind and one trusted value type in a storage version.
2. Address object identity has no durable meaning.
3. Namespace `pi` and every `pi.*` are reserved by contract; every built-in namespace starts with `pi.`, and application use is a trusted-programming defect.
4. Exactly five built-in prefix constructors encapsulate Branch inventory and operation cleanup grammar; their results are consumed only by namespace-scoped `scanValues()`.
5. Scalar and list addresses must not occupy the same physical location; this is a trusted-programming rule, not a runtime cross-kind collision check.
6. Typed reads and helper-constructed writes preserve `T`.
7. Scalar helpers reject list addresses; list helpers reject scalar addresses.
8. A Session/Storage operation never requires a second key after address construction.
9. Every list element is immutable and carries its globally unique committed write `seq`.
10. Elements at one list address are returned in sequence order on every backend.
11. Append performs no read of the target list.
12. Scalar/list writes are atomic with entries and usage in the same transaction.
13. Whole-list delete leaves no elements at that address.
14. Missing and empty lists both read as `[]`.
15. Base restore depends only on required scalar state and never enumerates auxiliary lists.
16. Auxiliary lists never establish effect completion or select a restart state.
17. JSONL compaction preserves surviving element sequences.
18. Terminal cleanup leaves no operation-owned scalar values or lists.
## Required tests
### Address typing and identity
- `value<T>()` and `list<T>()` preserve their declared `T` invariantly;
- scalar reads infer the bound address's value type;
- list reads infer its element type;
- `setValue` rejects an incompatible value at compile time;
- `appendList` rejects an incompatible element at compile time;
- scalar helpers reject list addresses and list helpers reject scalar addresses;
- independently constructed equal addresses access the same durable location;
- incompatible definitions of one physical address are documented/tested as a programming defect;
- empty keys work, while empty namespaces and separator-containing components reject;
- core and application code use the same `value()` and `list()` constructors, with no private constructor, privilege token, registry, or catalog;
- built-in address constructors produce exact `pi.branch.tip`, `pi.lane.*`, `pi.op.*`, `pi.pending.*`, `pi.session.name`, and `pi.entry.label` namespace/key/kind triples;
- every built-in namespace starts with `pi.`, while application fixtures use non-reserved namespaces;
- `branchTipInventoryPrefix()` binds the empty-key `pi.branch.tip` inventory prefix and is used only to enumerate Branches through `scanValues`;
- tool-args prefixes cover one operation and optionally one step, tool-memo prefixes cover one operation and optionally one invocation, preparation and tool-output prefixes cover exactly one operation;
- each prefix constructor result is used only by `scanValues`, and no inventory or cleanup call constructs a raw reserved namespace;
- application addresses work without declaration merging or core catalogs;
- no Storage or Session operation accepts an additional key argument.
### Scalar regression
- set/get/delete/recreate behavior is unchanged;
- replacement retains only the latest logical value and latest set `seq`;
- typed write helpers preserve mixed transaction order;
- prefix scans interpret the bound address key as a prefix and remain namespace-scoped;
- new scalar JSONL/SQLite files use only the value/list schema; pre-WP01 WIP files are explicitly unsupported.
### List conformance
Extend the shared backend conformance suite:
- append one element and page it;
- multiple appends to one address in one transaction;
- appends separated by unrelated writes preserve per-list order;
- every element receives its own global write `seq`;
- ascending and descending exclusive cursors;
- default, explicit, invalid, and capped limits;
- absent list returns `[]`;
- whole-list delete and delete of absent list;
- delete followed by append in one transaction;
- rollback when a later write is invalid;
- atomic list + entry + usage + scalar transaction;
- identical pages and cursors on Memory, JSONL, and SQLite;
- JSONL torn multi-write transaction exposes no list element;
- JSONL replay and compaction preserve cursors;
- SQLite paging uses the primary key without temporary sorting;
- append performs no list read;
- base restore constructs trusted scalar projection without list reads, followed by bounded consumption-time hydration;
- close rejects later reads and honors already-admitted commits.
### Application surface
- an application-wide scalar value requires no extra key at get/set;
- an application-wide list requires no extra key at read/append;
- an application can construct dynamic per-workspace addresses explicitly;
- Storage and Session accept the same address objects and infer the same types;
- direct Session writes serialize and commit once;
- explicit `Session.mutate()` can atomically combine typed value/list writes with entries and usage.
### Assistant-frame integration — deferred beyond WP01
- every converted non-terminal frame appends under the exact bound effect-pending response address;
- terminal `done`/`error` events append nothing;
- appends enqueue synchronously without provider backpressure;
- every frame-write promise has an observed fault path;
- only the latest promise reference is retained for settlement ordering;
- awaiting the latest promise implies every earlier append completed;
- reduced pages reconstruct the same partial message as uninterrupted streaming;
- missing list restores as no durable partial;
- final/synthetic settlement atomically deletes the frame list;
- unknown-effect recovery reads only the bounded list derived from current scalar state;
- external finalization deletes the operation-owned list;
- idle forks contain no frame list;
- backend byte growth is append-linear rather than repeated-snapshot growth.
## Implementation map
Expected primary changes:
- replace `session/registers.ts` with `packages/agent/src/harness/session/values.ts` containing addresses, constructors, typed write helpers, and built-in address constructors;
- remove `RegisterValues`, namespace unions, register token types, and raw namespace/key read signatures from `session/types.ts`;
- expose `ValueReader` through Storage, SessionReader, SessionMutator, and Session;
- expose direct application scalar/list methods on Session using bound addresses;
- update Memory state, JSONL codec/storage, snapshots, fork/rewrite code, instrumentation, and conformance suites;
- replace SQLite's unfinished initial schema in place with `scalar_values` and `list_values`; keep storage version 1 and add no migration runner;
- update telemetry schema sources and regenerate `telemetry-schema.md`; do not edit that generated file manually.
WP01 stops after generic addresses/storage and projection-only restore coverage. `assistant-durability.md` specifies the later consuming lifecycle; assistant execution, deferred polling, recovery, snapshot hydration, memo/checkpoint capabilities, and operation cleanup land only with their runtime work packages.

83
third_party/pi/packages/agent/docs/work-packages/00-runtime1-removal.md vendored
View File

@@ -1,83 +0,0 @@
# WP00 — Runtime1 removal
## Status
Complete. No tag. Runtime2 is the sole public implementation. Do not release while its execution path is incomplete.
## Goal
Make runtime2 the sole public harness implementation, delete runtime1 and its obsolete tests, then stop before adding runtime2 behavior.
## Prerequisites
- The approved acceptance/hook redesign and durability handoffs are present in `harness.md`, `values.md`, `assistant-durability.md`, and `tool-durability.md`.
- Existing tests are evidence, not authority.
## Work, in order
1. **Reconcile the contract.** Fold the approved acceptance/hook redesign into `harness.md`, including durable `starting`, atomic hook-free acceptance, driver-owned `before_run`, `before_drive`, request-local system-prompt transformation, trusted restore, and removal of process-origin activation semantics. Audit §§0.4, 1.2, 3.1–3.6, 4.1–4.2, 4.5, 5.1–5.2, 5.5–5.6, and Parts 8–9; remove every stale `BeforeResumePrepared`, `before_resume`, `resumeData`, `systemPromptOverride`, stable-ID routing, reservation, and `fresh | continue | resume` activation reference. Delete obsolete runtime planning documents once `harness.md` and linked handoffs own the active contract.
2. **Harvest before deletion.** Inspect `agent-harness-runtime.test.ts`, `agent-harness-r2/r3/r4.test.ts`, and old `restore.test.ts`. Preserve unique scenarios in the detailed future rows or a temporary categorized inventory; explicitly discard old reservation, `before_resume`, `resumeData`, persisted hook prompt override, semantic restore audit, and pre-`outcome_ready` tool-crash behavior.
3. **Remove runtime1-only public members.** Delete `before_resume`, `BeforeResumePrepared`, `resumeData`, `systemPromptOverride`, and stable hook-ID routing. Add the approved `before_drive` and `transform_context` shapes. Update telemetry schema source and regenerate its document. Do not implement `starting` or acceptance behavior here.
4. **Switch the factory.** Add `packages/agent/src/harness/runtime2/index.ts`, point `agent-harness.ts` at it, and add a constructor-selection regression. Verify the experimental coding-agent worker still creates the harness, subscribes to events, and closes it.
5. **Delete runtime1 source and tests** listed below.
6. **Update `[Unreleased]` on `main` or a pull-request branch** for the public breaking removals and temporarily incomplete factory. Repository policy forbids changelog edits on `dev`, so WP00 records but does not perform that release-facing step here.
7. Run the retained tests and checks. Fix every failure; do not restore compatibility shims.
## Delete
```text
packages/agent/src/harness/runtime/**
packages/agent/src/harness/restore.ts
packages/agent/test/harness/agent-harness-runtime.test.ts
packages/agent/test/harness/agent-harness-r2.test.ts
packages/agent/test/harness/agent-harness-r3.test.ts
packages/agent/test/harness/agent-harness-r4.test.ts
packages/agent/test/harness/restore.test.ts
packages/agent/test/harness/scratch/r1.ts
packages/agent/test/harness/scratch/r2.ts
packages/agent/test/harness/scratch/r3.ts
packages/agent/test/harness/scratch/r4.ts
```
## Retain
- all `test/harness/runtime2/**` tests;
- Session, Branch, storage, repository, backend-conformance, and instrumentation tests;
- execution assistant/tool/primitives tests;
- config, hooks, events, telemetry, compaction, and branch-summary code/tests;
- `types.test.ts`, updated for the reduced public contract;
- `packages/agent/src/agent-loop.ts` unchanged.
Do not parameterize obsolete runtime1 suites against runtime2 and do not keep a runtime1 smoke suite.
## Acceptance
- No source import references `harness/runtime/*`.
- Public `AgentHarness.create()` selects runtime2.
- Runtime2 creation, events, inspection, close, and fault tests pass.
- These coding-agent tests pass:
- `experimental-remote-runtime.test.ts`
- `experimental-session-worker-manager.test.ts` (the upstream replacement for the removed `experimental-session-worker.test.ts`)
- `experimental-session-worker-lifecycle.test.ts`
- Every modified test passes individually.
- Agent and root TypeScript pass.
- `git diff --check` and `npm run check` pass.
## Outcome
- The accepted hook/drive contract is normative in `harness.md`; stale acceptance/resume contracts are absent.
- Runtime1 source, validating restore, obsolete suites, and R1–R4 scratch scenarios are deleted.
- `AgentHarness.create()` resolves through `runtime2/index.ts`; a constructor-selection regression proves it.
- The scenario harvest added missing tool-close, identity-preflight, recovery-ordering, turn-bracket, and telemetry cases to future rows.
- Upstream added two real remote prompt tests after this handoff was drafted. They remain present but skipped with an R2 re-enable requirement because runtime2 execution is intentionally incomplete. Worker creation, attachment, lifecycle, operation correlation, and close coverage passes.
## Non-goals
- No bound-value/list implementation.
- No implementation of `starting` or atomic acceptance.
- No drive owner, provider, retry, deferred, or tool execution.
- No runtime1 parity work, compatibility layer, archaeology tag, or release.
## Stop condition
Stop once runtime1 is absent, runtime2 is the public factory, retained coverage is green, and all checks pass. Report the deletion and harvested scenarios; do not begin another work package.

327
third_party/pi/packages/agent/docs/work-packages/01-bound-values-lists.md vendored
View File

@@ -1,327 +0,0 @@
# WP01 — Bound values and lists
## Status
Complete. `harness.md` is normative. [`values.md`](../values.md) supplies the detailed address, backend, and conformance design.
## Goal
Replace the retained register/custom-state storage surface with bound `Value<T>` and `ValueList<T>` addresses across Session, Memory, JSONL, SQLite, instrumentation, tests, and public application access. Stop before runtime execution consumers.
## Decisions fixed for this package
1. **Core and application namespaces.** Every core address uses the exact documented `pi.*` namespace. Applications use their own non-reserved namespaces through the same public `value()` / `list()` constructors. `fact.custom` and its API are deleted, not renamed to a built-in custom namespace.
2. **Forks.** Generic forks copy only explicitly handled core addresses: Branch tip/lane configuration plus fresh lane state, session name, and labels whose targets copy. They copy no `pi.op.*`, `pi.pending.*`, list, ledger, or arbitrary application address. A later application feature must add an address-specific fork policy before relying on copied application state.
3. **Trusted kind discipline.** Using one `(namespace, key)` as both a value and a list is a trusted-programming defect. Backends do not add cross-kind collision checks, triggers, registries, or catalogs.
4. **Operation names.** Keep source `OperationMeta` for immutable acceptance metadata and `Operation` for the process-local `{ meta: OperationMeta, state: OperationState }` projection. `operationMeta(id)` binds `Value<OperationMeta>`; the composite is never persisted as one value.
5. **Query ordering and bounds.** `scanValues()` returns key-ascending results. `readList()` limits only one query page, never total list length or bytes: reject non-positive or non-safe limits, default to 1,000, and clamp larger values to 10,000.
6. **No WIP compatibility.** Replace the unfinished format-4 storage schema in place. JSONL remains format 4/storage version 1 but accepts only the new value/list records. SQLite keeps `SQLITE_STORAGE_VERSION = 1`, edits `001_initial.sql` in place, renames `registers` to `scalar_values`, and adds `list_values`. Pre-WP01 format-4 JSONL and SQLite files are unsupported. Add no migration runner or legacy decoder.
7. **Generic infrastructure only.** WP01 defines every built-in value/list constructor, including future assistant/tool addresses, but does not implement their runtime consumers.
## Public and storage contract
Add `packages/agent/src/harness/session/values.ts` with:
- invariant `Value<T>` and `ValueList<T>` address types;
- universal `value<T>(namespace, key?)` and `list<T>(namespace, key?)` constructors;
- namespace/key validation only: non-empty namespace and no `\u0000` component;
- `StoredValue<T>`, `ListElement<T>`, `ListCursor`, and `ListReadOptions`;
- typed `setValue`, `deleteValue`, `appendList`, and `deleteList` write helpers using `NoInfer<T>`;
- every exact built-in constructor and the five scan-prefix constructors from `values.md`.
Replace the old API throughout:
```ts
getRegister(namespace, key) -> getValue(address)
listRegisters(namespace, prefix) -> scanValues(prefixAddress)
register set/delete writes -> typed value helpers
```
Storage and the historical Session reader, mutator, tree-view, and repository surfaces expose the same bound-address reads:
```ts
getValue<T>(address: Value<T>): Promise<StoredValue<T> | undefined>;
scanValues<T>(prefix: Value<T>): Promise<StoredValue<T>[]>;
readList<T>(address: ValueList<T>, options?: ListReadOptions): Promise<ListElement<T>[]>;
```
The historical tree-view and Session surfaces additionally expose one-commit direct writes:
```ts
setValue<T>(address: Value<T>, next: NoInfer<T>): Promise<void>;
deleteValue<T>(address: Value<T>): Promise<void>;
appendList<T>(address: ValueList<T>, element: NoInfer<T>): Promise<void>;
deleteList<T>(address: ValueList<T>): Promise<void>;
```
`SessionMutator` retains one explicit `commit(writes)` and does not gain direct committing methods. Each write array composes helper-constructed value/list writes with entries and usage.
Keep `getName` / `setName` and `getLabel` / `setLabel` as wrappers over `sessionName` and `entryLabel(id)`. Delete `getCustomFact` / `setCustomFact`. Rename the public passive metadata event from `fact_update` to the `value_update` shape already specified by `harness.md`; it covers only session-name and entry-label wrappers, not arbitrary application writes.
## Files
### Add
- `packages/agent/src/harness/session/values.ts`
- `packages/agent/test/harness/values.test.ts`
- `packages/session-backends/sqlite-node/src/sqlite/session/values.ts`
### Delete or rename
- delete `packages/session-backends/sqlite-node/src/sqlite/session/registers.ts` after moving its scalar behavior to `values.ts`;
- remove all register/global-map/custom-fact declarations from `packages/agent/src/harness/session/types.ts`.
### Agent source
- `packages/agent/src/harness/agent-harness.ts`
- `packages/agent/src/harness/session/types.ts`
- `packages/agent/src/harness/session/commit.ts`
- `packages/agent/src/harness/session/storage-state.ts`
- `packages/agent/src/harness/session/memory.ts`
- `packages/agent/src/harness/session/session.ts`
- `packages/agent/src/harness/session/fork.ts`
- `packages/agent/src/harness/session/index.ts`
- `packages/agent/src/harness/session/jsonl/storage.ts`
- `packages/agent/src/harness/session/jsonl/repo.ts`
- `packages/agent/src/harness/session/testing/storage-decorator.ts`
- `packages/agent/src/harness/session/testing/instrumented-storage.ts`
- `packages/agent/src/harness/session/testing/types.ts`
- `packages/agent/src/harness/session/testing/conformance/storage.ts`
- `packages/agent/src/harness/session/testing/conformance/session-repo.ts`
- `packages/agent/src/harness/session/testing/benchmark/storage.ts`
- `packages/agent/src/harness/session/testing/benchmark/session-repo.ts`
- `packages/agent/src/harness/session/testing/index.ts`
- `packages/agent/src/harness/runtime2/restore.ts`
- `packages/agent/src/harness/runtime2/harness.ts`
- `packages/agent/src/harness/runtime2/lane.ts`
- `packages/agent/src/harness/telemetry.ts`
- `packages/agent/src/index.ts` and `packages/agent/src/node.ts` only as needed to verify the new public exports; do not add a second export path.
### Agent tests and generated documentation
- `packages/agent/test/harness/memory-storage.test.ts`
- `packages/agent/test/harness/memory-conformance.test.ts`
- `packages/agent/test/harness/memory-session-repo.test.ts`
- `packages/agent/test/harness/jsonl-storage.test.ts`
- `packages/agent/test/harness/jsonl-storage-conformance.test.ts`
- `packages/agent/test/harness/jsonl-session-repo.test.ts`
- `packages/agent/test/harness/jsonl-session-repo-conformance.test.ts`
- `packages/agent/test/harness/storage-backed-session.test.ts`
- `packages/agent/test/harness/session-tree.test.ts`
- `packages/agent/test/harness/session-create-lane.test.ts`
- `packages/agent/test/harness/instrumented-storage.test.ts`
- `packages/agent/test/harness/types.test.ts`
- `packages/agent/test/harness/telemetry.test.ts`
- `packages/agent/test/harness/runtime2/harness.test.ts`
- `packages/agent/test/harness/runtime2/lane.test.ts`
- `packages/agent/test/harness/runtime2/restore.test.ts`
- regenerated `packages/agent/docs/telemetry-schema.md`
### SQLite backend
- `packages/session-backends/sqlite-node/src/sqlite/migrations/001_initial.sql`
- `packages/session-backends/sqlite-node/src/sqlite/repo.ts`
- `packages/session-backends/sqlite-node/src/sqlite/session.ts`
- `packages/session-backends/sqlite-node/src/sqlite/storage.ts`
- `packages/session-backends/sqlite-node/src/sqlite/index.ts` if needed for the renamed module
- `packages/session-backends/sqlite-node/test/storage.test.ts`
- `packages/session-backends/sqlite-node/test/storage-conformance.test.ts`
- `packages/session-backends/sqlite-node/test/repo.test.ts`
- `packages/session-backends/sqlite-node/test/adapter.test.ts`
- `packages/session-backends/sqlite-node/test/sql.test.ts`
### Coding-agent consumer
- `packages/coding-agent/test/experimental-session-support.ts`
- verify `packages/coding-agent/test/experimental-remote-runtime.test.ts`
- verify `packages/coding-agent/test/experimental-server-replacement.test.ts`
If the final old-API grep identifies another retained source/test call site, it belongs to WP01; do not add a compatibility shim to avoid touching it.
## Work, in order
1. **Add the address vocabulary.** Implement `values.ts`, export it through the existing session/root path, and add focused compile-time/runtime address tests. Include exact built-in namespace/key/kind tests and prefix-constructor tests before migrating callers.
2. **Cut the shared API once.** Replace register types and writes in `types.ts`/`commit.ts`; split `StorageState` into current scalar values and surviving list elements; implement Memory reads/writes, ordered prefix scans, paged list reads, transaction validation/application, snapshots, and direct Session methods. Remove custom-fact APIs and migrate name/label wrappers.
3. **Migrate JSONL and generic fork/snapshot code.** Encode only `kind:"value"` and `kind:"list"`; replay set/delete/append/delete; preserve transaction-line torn-tail atomicity; serialize surviving list elements with original `seq` merged in global sequence order; preserve the sequence high-water mark. This extends existing snapshot serialization only—do not add a new compaction trigger or precise-rewrite feature. Forks copy the fixed core set from Decisions item 2, re-sequence destination scalar values after copied entries as today, and copy no lists.
4. **Migrate instrumentation, conformance, and benchmarks.** The storage decorator exposes all three reads; instrumented storage records erased value/list writes in exact order without content telemetry. Extend shared conformance before backend-specific assertions.
5. **Migrate runtime2 shell call sites.** Replace lane/harness raw writes with built-in constructors/helpers. `restore.ts` uses `scanValues(branchTipInventoryPrefix())` plus exact `getValue` lookups and performs no `readList()` call. Do not add acceptance, drive, hydration, or cleanup behavior.
6. **Replace the SQLite WIP schema and adapter.** Edit `001_initial.sql` in place, implement scalar operations and indexed list append/delete/paging in `session/values.ts`, keep every write inside the existing `BEGIN IMMEDIATE` writer-lease transaction, update both fork snapshot paths, and retain all entry/usage/branch/lease behavior from current `dev`.
7. **Migrate public events, tests, and the coding-agent helper.** Remove old type assertions and raw namespaces. Change `fact_update` to `value_update`. Keep the two remote prompt tests skipped for the existing WP00 reason; WP01 must not alter runtime execution.
8. **Update telemetry and documentation.** Change `pi.session.write` item kinds from `register` to `value` and `list`, regenerate `telemetry-schema.md`, run the old-API sweeps, and record any branch-policy-deferred changelog requirement. Do not edit a changelog on `gramps` unless it becomes a pull-request branch or the user requests it.
## Backend requirements
### Memory
- prepare and validate the complete transaction before mutating entries, values, lists, usage, or stats;
- current scalar replacement stores only the latest value and set `seq`;
- list append performs no list read and stores each global write `seq`;
- list delete removes the whole exact key;
- snapshots include current scalar values and surviving list elements.
### JSONL
- keep format 4/storage version 1 with no legacy register decode;
- one single-write object or multi-write array remains one atomic line;
- replay produces the same logical state as Memory;
- torn final lines expose no partial transaction;
- snapshot serialization preserves surviving list-element sequences and the next-sequence high-water mark.
### SQLite
Use:
```sql
CREATE TABLE scalar_values (
namespace TEXT NOT NULL,
key TEXT NOT NULL,
seq INTEGER NOT NULL,
value TEXT NOT NULL,
PRIMARY KEY (namespace, key)
) WITHOUT ROWID;
CREATE TABLE list_values (
namespace TEXT NOT NULL,
key TEXT NOT NULL,
seq INTEGER NOT NULL,
value TEXT NOT NULL,
PRIMARY KEY (namespace, key, seq)
) WITHOUT ROWID;
```
Ascending and descending list queries use the primary key with an exclusive sequence predicate and `LIMIT`. Add `EXPLAIN QUERY PLAN` assertions proving no table scan or temporary ordering b-tree. Do not change storage version, add migrations, or weaken the current lease/fence/fork behavior.
## Required coverage
### Address and type tests
- invariant address typing and inferred scalar/list result types;
- `NoInfer` rejects incompatible set/append values;
- scalar helpers reject list addresses and list helpers reject scalar addresses;
- independently constructed equal addresses resolve the same location;
- empty key works; empty namespace and `\u0000` components reject;
- exact built-in namespaces/key grammars and exactly five prefix constructors;
- application-wide and dynamic non-reserved addresses require no second operation-time key;
- no registry, catalog, privilege constructor, global value map, or runtime `pi.*` gate.
### Shared scalar/list conformance
- scalar set/get/delete/delete-absent/recreate and latest set `seq`;
- namespace-scoped, key-ascending prefix scans;
- append one and several elements, including several appends in one transaction;
- appends separated by unrelated writes preserve per-list order and global element sequences;
- ascending/descending exclusive cursors;
- default, explicit, invalid, and clamped query-page limits;
- absent list, whole-list delete, delete-absent, and delete-then-append;
- atomic entry + usage + value + list transactions;
- rollback when any sibling write is invalid;
- no list read on append;
- close rejects later reads while already-admitted commits drain.
Do not add a value/list collision test: cross-kind misuse is intentionally an unenforced trusted-programming defect.
### Backend and repository tests
- identical Memory/JSONL/SQLite pages and cursors;
- JSONL single/multi-write replay, torn-tail behavior, and sequence-preserving snapshot output;
- SQLite query plans and writer-lease transaction behavior;
- branch/tree forks copy session name, eligible labels, and lane configuration/Branch tip with fresh lane state;
- forks exclude operation/pending values, all lists, application addresses, last results, queues, and ledger rows;
- repository parent metadata, entry IDs, stats, branch indexes, v3 normalization, UUIDv7/follower IDs, and current SQLite lease/fork scenarios remain unchanged;
- runtime2 restore enumerates lanes through the one prefix constructor and reads no list.
## Deferred consumers
The following `values.md` requirements are explicitly not WP01 coverage:
- assistant frame conversion, append scheduling, settlement, recovery, cancellation, snapshot hydration, and byte-growth tests (R2/R3/R6/R12);
- invocation `getMemo` / `setMemo`, tool-output checkpoint writes, outcome cleanup, and prefix-driven operation cleanup (R4/R6);
- any consumption-time list hydration beyond proving base restore reads no list;
- runtime acceptance, driving, provider/tool effects, or operation-state redesign.
WP01 still exports `pendingAssistantFrames`, `operationToolMemo`, `pendingToolOutput`, and every cleanup prefix so later packages do not redesign storage.
## Removal checks
These must have zero matches in retained source/tests, excluding immutable released changelog history and archived prose:
```bash
rg -n 'getRegister\(|listRegisters\(|RegisterValues|RegisterNamespace|RegisterSetWrite|\bRegister<' \
packages/agent packages/session-backends/sqlite-node packages/coding-agent \
--glob '!**/dist/**' --glob '!**/CHANGELOG.md' --glob '!**/docs/**'
rg -n 'kind: "register"|getCustomFact\(|setCustomFact\(|fact\.(name|label|custom)|fact_update' \
packages/agent packages/session-backends/sqlite-node packages/coding-agent \
--glob '!**/dist/**' --glob '!**/CHANGELOG.md' --glob '!**/docs/**'
rg -n '"(branch\.tip|lane\.(config|state|lastResult)|op\.(meta|state|tool_args|preparation)|pending\.entry)"' \
packages/agent packages/session-backends/sqlite-node packages/coding-agent \
--glob '!**/dist/**' --glob '!**/CHANGELOG.md' --glob '!**/docs/**'
rg -n '\bregisters\b' packages/agent/src/harness/session packages/session-backends/sqlite-node/src \
--glob '*.ts' --glob '*.sql'
rg -n '"register"' \
packages/agent/src/harness/telemetry.ts \
packages/agent/test/harness/telemetry.test.ts \
packages/agent/docs/telemetry-schema.md
```
Do not treat unrelated model/provider/hook registration terminology as storage API residue.
## Validation
Run each created or modified test file directly and iterate until green. At minimum:
```bash
# From packages/agent
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run \
test/harness/values.test.ts \
test/harness/memory-storage.test.ts \
test/harness/memory-conformance.test.ts \
test/harness/memory-session-repo.test.ts \
test/harness/jsonl-storage.test.ts \
test/harness/jsonl-storage-conformance.test.ts \
test/harness/jsonl-session-repo.test.ts \
test/harness/jsonl-session-repo-conformance.test.ts \
test/harness/storage-backed-session.test.ts \
test/harness/session-tree.test.ts \
test/harness/session-create-lane.test.ts \
test/harness/instrumented-storage.test.ts \
test/harness/types.test.ts \
test/harness/telemetry.test.ts \
test/harness/runtime2/harness.test.ts \
test/harness/runtime2/lane.test.ts \
test/harness/runtime2/restore.test.ts
# From packages/session-backends/sqlite-node
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run \
test/adapter.test.ts \
test/repo.test.ts \
test/sql.test.ts \
test/storage.test.ts \
test/storage-conformance.test.ts
# From packages/coding-agent
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run \
test/experimental-remote-runtime.test.ts \
test/experimental-server-replacement.test.ts
```
Then run from the repository root:
```bash
cd packages/agent && npm run check:telemetry-docs
cd "$(git rev-parse --show-toplevel)"
node_modules/.bin/tsgo --noEmit -p packages/agent/tsconfig.build.json
node_modules/.bin/tsgo --noEmit
git diff --check
npm run check
./test.sh
```
Never run unrestricted Vitest, `npm test`, paid-provider tests, or `npm run build`.
## Stop condition
Stop when every retained backend and Session surface uses bound values/lists; all core addresses use the exact `pi.*` grammar; arbitrary application addresses work but generic forks exclude them; old register/fact/custom-state APIs and physical names are absent; base restore performs no list read; schema/compatibility decisions above are implemented; focused, conformance, TypeScript, telemetry-doc, diff, and repository checks pass. Report the final schema and fork behavior. Do not begin runtime acceptance, assistant/tool consumers, or any later work package.

542
third_party/pi/packages/agent/docs/work-packages/02-atomic-run-acceptance.md vendored
View File

@@ -1,542 +0,0 @@
# WP02 — Atomic acceptance and coherent lane observation
## Status
Complete. Phase A established minimal attachment, open-operation inventory, Session-line inspection/watch capture, commit-continuation recipient binding, and in-band identity-failure vocabulary. Phase B landed in `beac75ecc` with focused, monorepo-check, full-suite, and final Fable review passing.
Implementation also updated downstream protocol/coding-agent wire projections required by the changed public contract. No execution owner, provider/tool effect, timer, retry, polling, cancellation procedure, manual action, or terminal settlement was added.
## Goal
Deliver two effect-free boundaries:
```text
idle lane
→ atomic prompt/skill/template acceptance
→ durable open operation in payload-free starting
open or running lane
→ Session-line watch capture
→ complete snapshot plus gap-free subsequent events
```
After `AgentHarness.create(options, context)` succeeds:
- every configured lane has a complete small process-local projection;
- `open` inventories every durable current operation without predicting model/tool availability;
- attachment starts no hook, provider, tool, timer, breakpoint, drive owner, or application callback;
- `inspectExecution(context)` observes the small projection and local owner on the Session line;
- `watch(context)` registers buffering, clones live presentation, and performs bounded snapshot reads in one no-write lane job;
- every committing lane job publishes memory and synchronously binds its event batch in the commit continuation;
- the mutation does not await listener delivery, while the public operation does.
WP02 does not implement drive, provider generation, hooks, tools, retries, deferred polling, cancellation procedures, manual action execution, or terminal settlement.
## Decisions
### 1. Attachment restores projection, not presentation
Passing the open Session to `AgentHarness.create()` transfers orchestration ownership until create rejects or the harness closes. Direct `Session.mutate`, `Session.createLane`, reserved-address writes, and second-harness construction are prohibited during that interval, so lane inventory cannot race an out-of-band lane creation.
Attachment reads only:
- `branchTip`, `laneConfig`, `laneState`, optional `laneLastResult`;
- `operationMeta` and `operationState` for a current operation.
It validates required existence, operation id/lane ownership, and intent/state kind compatibility. Projection corruption faults `create()`.
Attachment does not read transcript, queues, pending writes, drained payloads, deferred sources, frames, tool calls, arguments, checkpoints, preparations, memos, or staged outcomes. Those references are checked by `watch()` or drive when consumed. Missing or contradictory required payload data is terminal storage corruption and faults that consumer. Optional frame/checkpoint absence is legal.
### 2. Watch owns detailed snapshot reads
One no-write Session mutation job defines the watch boundary:
```text
enter after all earlier lane jobs
→ synchronously register buffering watcher
→ synchronously clone live presentation state
→ perform bounded durable reads while later lane jobs are excluded
→ assemble snapshot
→ release Session line
→ return handle
```
There is no special first-watch cache. The same path handles immediate post-attachment watch, reconnect, and watch during live execution.
The bounded read set is:
- one compaction-bounded branch scan from the current tip;
- exact `pendingEntry(id)` reads for next-run, steer, follow-up, writes, and abort drains;
- exact deferred source entry when represented;
- tools-phase assistant entry and exact args for represented `effect_pending` calls, using `batch.turnId` as the args step id;
- bounded exact assistant-frame pages and exact optional tool checkpoints.
`ToolCall.sourceIndex` is the index in the assistant message's full content array, not a filtered tool-call ordinal. A represented call must index a tool-call block.
### 3. Recipient binding belongs in the commit continuation
A successful committing lane job performs:
```text
commit
→ publish small owned projection
→ synchronously bind recipients and append the complete `{ event, context }` batch
→ return from the mutation without awaiting delivery
→ await delivery before the public operation resolves
```
WP02 initially implemented this with `enqueue()` plus caller-operated `start()`. WP04 replaces that gate with one immediate `emitBatch()` call in the same commit-observation continuation. Recipient binding and delivery awaiting remain unchanged. A listener or watcher registered after `emitBatch()` cannot receive that historical event.
The only watcher/publication orders are:
```text
watcher first
→ snapshot-before + complete buffered event batch
publication/`emitBatch` first
→ snapshot-after + no old event
```
A live provider/tool presentation update follows the same synchronous publish-plus-`emitBatch` discipline. Frame/checkpoint commits are lane jobs and queue behind watch capture.
### 4. Inspection is a no-write Session-line observation
`inspectExecution(context)` observes:
- lane and tip;
- configured model identity as unresolved `{ provider, modelId }` strings;
- current operation id/kind/start time;
- process status `running`, `open`, or durable `aborting`;
- captured model identity when the current durable phase contains one;
- optional latest result.
It does not resolve model/tool registries and does not read transcript or presentation payloads.
### 5. Missing implementations are in-band outcomes
There is no `blocked`, missing-identity suspension, predictive classifier, or acceptance registry preflight.
At the actual execution boundary:
- unavailable captured model or configured active-tool definition before provider intent becomes a non-retryable configuration failure;
- pre-intent configuration failure reserves no response/usage ids and fabricates no assistant response or usage row;
- restored `effect_pending` settles uncertainty under its existing reserved ids before any later configuration failure;
- unavailable deferred model durably abandons redemption through configuration failure; R7 implements and tests restored `effect_pending` abandonment, including deletion of its exact old assistant-frame list while dropping the reserved response/usage strings without fabricating settlement;
- missing requested tool stages a direct `isError` `ToolResultMessage` and continues;
- missing/no-longer-safe replay implementation synthesizes interruption instead of waiting.
Synthetic harness tool results omit `details`. A tool owns the type of its details contract; the harness must not invent `{}` or a diagnostic object. `isError` and human-readable content carry the tool-level diagnosis. Run-level configuration failure remains machine-readable through `OperationError` and `laneLastResult`.
Stable configuration error codes:
- `model_unavailable`, details `{ provider, modelId }`;
- `configured_tools_unavailable`, details `{ tools: string[] }`.
`failure_drain` gains `{ kind: "configuration" }` provenance. Actual transitions land with their owning execution packages; WP02 lands the normative/source vocabulary only.
### 6. Acceptance is independent of process registries
Acceptance validates durable caller input and lane state, not current model/tool registrations. This avoids a time-of-check/time-of-use check and permits acceptance in one process followed by execution in another.
A misconfigured convenience prompt eventually returns a durable failed run from drive. Explicit hosted acceptance remains durable even before an execution worker loads implementations.
### 7. Invocation Context remains explicit
Every current public harness/lane operation receives trailing `context: Context`. Acceptance, attachment, watch capture, Session reads/commit, faults, and event publication preserve it. Shared harness/lane/Session receivers retain no caller Context. Context and its signal/telemetry values are never durable business data.
Buffered events retain the exact emitting Context. Invocation cancellation remains distinct from durable `requestAbort()`.
## End-state public contract
```ts
export interface ModelIdentity {
provider: string;
modelId: string;
}
export type OperationStatus = "running" | "open" | "aborting";
export interface OpenOperation {
lane: string;
operationId: string;
kind: "run" | "compaction" | "navigation";
startedAt: number;
aborting?: true;
}
export interface CurrentOperationInfo {
id: string;
kind: "run" | "compaction" | "navigation";
startedAt: number;
status: OperationStatus;
capturedModel?: ModelIdentity;
}
export interface LaneExecutionInfo {
lane: string;
tipId: string | null;
configuredModel: ModelIdentity;
current: CurrentOperationInfo | null;
lastResult?: LaneLastResult;
}
export interface LaneInfo {
name: string;
tipId: string | null;
operation: CurrentOperationInfo | null;
}
export interface AgentHarnessConstructor {
create<TContext extends object | undefined = object | undefined>(
options: AgentHarnessOptions<TContext>,
context: Context,
): Promise<{ harness: AgentHarness<TContext>; open: OpenOperation[] }>;
}
```
Rules:
- `open` has exactly one item per durable current operation and omits idle lanes;
- `aborting:true` comes only from durable `cancel_requested`;
- `open` is inventory, not scheduling or identity advice;
- normal applications establish watch and call `resume(context)`;
- hosted schedulers retain expected-id `drive` fencing;
- configured/captured identity fields are durable strings and may not resolve.
### Outcomes
Delete `MissingIdentitySuspension`, `MissingIdentities`, missing-identity drive waiting, and missing-identity suspension events.
Keep deferred suspension as provider semantics:
```ts
{ kind: "suspended"; reason: "deferred"; ... }
```
WP05 removes the withdrawn action outcome before execution is enabled. Convenience operation outcomes remain operation-tagged branches of `ResumeOutcome`.
### Snapshot
```ts
export interface LaneSnapshot {
lane: string;
transcript: Entry[];
tipId: string | null;
lastResult?: LaneLastResult;
operation: null | {
id: string;
kind: "run" | "compaction" | "navigation";
startedAt: number;
status: OperationStatus;
action?: ActionInfo;
retry?: { attempt: number; maxAttempts: number; nextAttemptAt: number };
deferred?: { handle: DeferredHandle; poll: number };
drained?: { steer: QueuedItem[]; followUp: QueuedItem[] };
streamingMessage?: AssistantMessage;
runningTools: {
toolCallId: string;
toolName: string;
args: unknown;
partialResult?: AgentToolResult<unknown>;
}[];
};
queues: { steer: QueuedItem[]; followUp: QueuedItem[]; nextRun: QueuedItem[] };
pendingWrites: {
entryId: string;
type: EntryType;
customType?: string;
message?: AgentMessage;
data?: JsonValue;
}[];
faulted: boolean;
}
```
Configuration is not duplicated in snapshots. `inspectExecution()` exposes configured/captured model identities; getters expose current configuration.
## Atomic run acceptance
WP02 implements `accept()` for prompt, skill, and prompt-template requests. Compaction/navigation acceptance remains with their execution packages.
State-independent normalization occurs before `Lane.command(plan, context)`:
- prompt strings/images;
- supplied message or message array;
- explicit skill formatting;
- prompt-template formatting;
- pending assistant rejection;
- unknown skill/template errors;
- supplied or minted operation and prompt-entry ids.
Public prompt convenience overloads remain exactly `[text, images | undefined, context]` and `[messageOrMessages, context]`, but convenience implementations remain `SliceNotImplemented` until R2.
Inside one lane command:
1. reject busy;
2. capture current `pendingNextRun` ids;
3. read and validate captured pending messages;
4. reject zero placed messages;
5. parent captured next-run entries before request prompt entries;
6. commit exactly once;
7. publish the small owned projection;
8. synchronously call `emitBatch` with the acceptance event batch and accepting Context;
9. return from the mutation callback without awaiting delivery;
10. await event delivery before `accept` resolves.
Exact writes:
```text
insert captured nextRun message entries
insert request prompt entries
delete captured pendingEntry values
set branchTip
set operationMeta
set operationState(run starting)
set laneState(current operation, pendingNextRun=[])
```
Exact event order:
```text
run_start
for each placed message:
message_start
message_end
entry_added
queue_update if nextRun was captured
```
Acceptance starts no drive or effect and writes no Context.
## Phase A — normative rewrite
Update `harness.md` before runtime source:
- replace eager attachment hydration with minimal projection restore;
- replace predictive status/classifier with open inventory;
- specify configured/captured identity inspection without resolution;
- move detailed reads to ad-hoc Session-line watch capture;
- require recipient binding in the exact commit-observation continuation;
- remove identity preflight/suspension/error/event types;
- specify in-band model/tool unavailability and configuration provenance;
- require direct detail-free synthetic tool-result messages;
- update invariants, races, roadmap, glossary, and Appendix C.
Review stop:
1. `git diff --check`;
2. fresh Terra contradiction/source-feasibility audit;
3. full-context Fable review against complete docs and source;
4. resolve all findings and repeat until no findings;
5. obtain user approval before runtime source.
## Phase B — implementation
### Public and durable types
Modify `agent-harness.ts`:
- remove `SuspendedOperation`, `MissingIdentityInfo`, `MissingIdentities`, and missing-identity outcome/event branches;
- add `ModelIdentity`, `OperationStatus`, `OpenOperation`, and corrected inspection/snapshot types;
- change create result from `suspended` to `open`;
- land operation-tagged action-required outcome families;
- correct `executeAction`/`runToCompletion` signatures;
- retain trailing Context everywhere.
Modify session types:
- add `failure_drain` configuration provenance;
- add the already-normative `ToolCall.outcome_ready` vocabulary without producers;
- document `sourceIndex` as full assistant-content index;
- add callback-scoped `scanBranch` to `SessionReader`; inheritance by `SessionMutator` and `Session` is intentional;
- implement it in `StorageBackedSession` and its mutator and `MemorySessionFacade`; Session mutation authority remains process-local and has no remote Session facade.
### Event publication
WP02 initially added synchronous recipient binding with a reserved delivery gate. WP04 supersedes only that mechanism:
- `HarnessEventBus.emitBatch()` snapshots ordinary and watcher recipients synchronously;
- delivery uses only that bound list;
- watcher buffers retain `{ event, context }`;
- a watcher registered after `emitBatch` receives nothing from that event.
`LaneCommand` commit decisions retain a synchronous post-commit event batch. After commit succeeds, `Lane.command` publishes `next`, calls `emitBatch` as its final mutation action, and carries the delivery promise outside `Session.mutate` before awaiting it. Every existing event-producing commit uses this path, including direct idle/pending appends, lane configuration setters, and session-name/entry-label setters.
WP04 also moves harness lane publication into `Session.createLane`'s committed-publication callback. Session commits, Harness publishes `lanesByName` and calls `emitBatch(lane_created)` in that continuation, and Session awaits the retained delivery promise after releasing the line.
Do not execute listeners on the line.
### Attachment
Keep `restore.ts` projection-only. Validate lane/operation ownership and kind compatibility. Remove `describeSuspension` and all payload hydration from `createAgentHarness`. Construct lanes and return `open` inventory without resolving registries.
### Inspection
Implement `inspectExecution(context)` as a no-write `Lane.command` observation. Derive captured model identity from the current durable phase. Read no storage payloads and resolve no registry identities.
### Watch
Implement `watch(context)` as one no-write lane job:
- register watcher and clone live presentation synchronously;
- perform the bounded read matrix above through callback-scoped readers;
- assemble isolated snapshot payloads;
- fault and unsubscribe on required corruption;
- retain legal optional absence;
- release the line before returning.
A focused internal snapshot helper is allowed. Do not create a persistent hydrated presentation cache or generic reducer.
### Out of scope
`drive`, `resume`, prompt convenience, compaction/navigation acceptance, abort/queues, `executeAction`, and `runToCompletion` remain `SliceNotImplemented` where not already implemented.
Provider/tool/configuration-failure transitions are specified now but implemented by R2/R3/R4/R7/R8. WP02 adds no effect, active operation, timer, hook, provider request, tool execution, retry, deferred fetch, cancellation reconciliation, or terminal transaction.
## Required tests
### Public types
- `SuspendedOperation`, `MissingIdentities`, and missing-identity status/outcomes/events are absent;
- open/current/status unions narrow exhaustively;
- configured and captured model identities are unresolved strings;
- action-required outcome families and method signatures match;
- prompt overload tuples remain exact;
- `AgentHarnessOptions` has no receiver telemetry default.
### Acceptance
- text-only, images-only, text-plus-images, supplied arrays;
- pending assistant rejection;
- skills/templates and unknown resources;
- empty input writes nothing unless captured nextRun supplies input;
- no identity registry preflight;
- supplied/minted operation ids;
- exact writes, parent chain, metadata, starting state, settings, commit materialization;
- pending-next-run capture/deletion;
- busy run/structural operation;
- one concurrent accept winner;
- commit failure and close races;
- exact event order and object-identical accepting Context;
- no hook, provider, tool, timer, drive-owner, or option callback invocation; passive event-listener delivery remains required.
### Attachment and inspection
- idle omitted and every open operation inventoried;
- durable cancellation sets only `aborting:true`;
- no transcript/pending/frame/tool reads at create;
- configured and captured model identities may differ and remain visible when unresolved;
- inspection runs on the Session line and performs no payload reads;
- projection corruption faults create;
- no option callback/effect starts.
### Event publication and watch
- recipient set binds at `emitBatch`: a watcher registered after publication but before delivery receives nothing;
- state/event batch publication happens before Session-line release;
- direct append, lane configuration, session-name/entry-label, acceptance, and lane-creation commits all use that publication path;
- `value_update` and `lane_created` bind recipients in their commit continuation, so later listeners receive neither historical event;
- watcher-first gives snapshot-before plus complete events;
- publication-first gives snapshot-after without old events;
- live update during awaited capture appears only as a buffered event after the cloned live snapshot;
- frame/checkpoint mutations queue behind capture;
- first watch and reconnect use the same path;
- exact transcript, queues, writes, drain, deferred, frame, tool args/checkpoint fields;
- required payload corruption faults watch and removes watcher;
- absent frames/checkpoints omit optional partials;
- pre-registration lifecycle is not replayed;
- buffered events retain object-identical emitting Context;
- payload mutation cannot affect later state/listeners;
- close/fault lifecycle matches the event bus.
### In-band identity vocabulary
Type/direct-state tests prove:
- acceptance has no `MissingIdentities` path;
- configuration failure provenance and stable error codes are representable;
- deferred configuration abandonment remains assigned to R7 rather than adding an execution transition here;
- missing-tool synthetic `ToolResultMessage` may omit `details`;
- sourceIndex uses full assistant-content indexing;
- outcome-ready calls are not rendered as running.
No execution transition is added in WP02.
## Files
### Add
- `packages/agent/test/harness/runtime2/accept.test.ts`
- focused watch tests if existing files become oversized.
### Modify
- `packages/agent/docs/harness.md`
- `packages/agent/docs/work-packages/02-atomic-run-acceptance.md`
- `packages/agent/src/harness/agent-harness.ts`
- `packages/agent/src/harness/events.ts`
- `packages/agent/src/harness/session/types.ts`
- `packages/agent/src/harness/session/session.ts`
- `packages/agent/src/harness/session/memory.ts`
- `packages/agent/src/harness/session/remote.ts`
- `packages/server/src/remote-session-manager.ts`
- `packages/agent/src/harness/runtime2/harness.ts`
- `packages/agent/src/harness/runtime2/lane.ts`
- `packages/agent/src/harness/runtime2/restore.ts`
- `packages/agent/src/harness/runtime2/types.ts`
- `packages/agent/test/harness/runtime2/harness.test.ts`
- `packages/agent/test/harness/runtime2/lane.test.ts`
- `packages/agent/test/harness/runtime2/restore.test.ts`
- `packages/agent/test/harness/types.test.ts`
- `packages/agent/test/harness/storage-backed-session.test.ts`
- `packages/agent/test/harness/memory-session-repo.test.ts`
- `packages/server/test/conformance.test.ts`
- event/session test files required by recipient binding and callback-scoped branch reads.
No backend schema, telemetry schema, coding-agent, or changelog change is expected. Stop for boundary review if one becomes necessary. On `dev`, defer changelog entries.
## Validation
After Phase A:
```bash
git diff --check -- \
packages/agent/docs/harness.md \
packages/agent/docs/work-packages/02-atomic-run-acceptance.md
```
After Phase B:
```bash
cd packages/agent
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run \
test/harness/runtime2/accept.test.ts \
test/harness/runtime2/harness.test.ts \
test/harness/runtime2/lane.test.ts \
test/harness/runtime2/restore.test.ts \
test/harness/types.test.ts
cd "$(git rev-parse --show-toplevel)"
git diff --check
npm run check
./test.sh
```
Report runtime2 source line counts. The synchronized pre-WP02 runtime2 baseline is 967 lines; treat growth above 1,900 source lines as a design review trigger, not a target.
## Stop condition
Stop when:
- acceptance commits exactly once into payload-free `starting` without registry preflight;
- attachment returns minimal complete projections and open inventory;
- inspection is a coherent Session-line no-write observation;
- watch captures detailed state ad hoc on the Session line;
- state publication and `emitBatch` recipient binding occur in the commit continuation while the mutation never awaits delivery;
- snapshots and buffered events have no gap or duplicate;
- required payload corruption faults its consumer;
- no execution effect or owner is introduced;
- focused tests, `npm run check`, and full tests pass;
- final Fable review has no findings.
Do not begin the first real drive package.

183
third_party/pi/packages/agent/docs/work-packages/03-remove-drive-deadlines.md vendored
View File

@@ -1,183 +0,0 @@
# WP03 — Remove drive deadlines
## Status
Complete. `DriveOptions.deadline` and the `DriveOutcome` `yielded` branch are removed from public types, active documentation, future package requirements, invariants, races, and tests. Obsolete `runtime2.md` is deleted. Focused tests, `npm run check`, and `./test.sh` pass.
WP02 is complete at `beac75ecc`. Preserve unrelated concurrent source work, especially current `packages/agent/src/harness/runtime2/lane.ts` changes, JSONL/fork work, plugins, RPC, and experimental directories.
## Problem
`DriveOptions.deadline` and the corresponding `DriveOutcome { kind: "yielded" }` do not provide a correctness boundary.
A deadline is checked only before starting another transition or effect. An admitted provider/tool/hook may run beyond it and the host may terminate the process anyway:
```text
check deadline
→ admit provider or tool
→ host limit expires while the effect is running
→ process dies with durable effect_pending
```
Unknown-outcome recovery remains mandatory. The deadline therefore does not prevent process loss, bound admitted work, make effects exactly once, or simplify recovery. Flue-style tool memoization depends on stable invocation ids and durable memos, not drive deadlines.
Deadline handling instead adds wall-clock policy to the durable core:
- a `yielded` public outcome unrelated to durable state;
- safe-boundary checks before hooks/effects/transitions;
- deadline-versus-retry-timer arbitration;
- deadline-versus-effect-admission races;
- convenience loops and event-bracket behavior for yields.
The host already owns scheduling and termination. Process loss is a controlled crash boundary recovered from durable operation state.
## Decision
Remove drive deadlines completely:
```ts
interface DriveOptions {
operationId: string;
waitForRetry?: boolean;
pollDeferred?: boolean;
}
type DriveOutcome =
| { kind: "settled"; operationId: string; outcome: TerminalOperationOutcome }
| { kind: "waiting"; operationId: string; reason: "retry"; notBefore: number }
| { kind: "waiting"; operationId: string; reason: "deferred"; deferred: DeferredHandle }
| { kind: "action_required"; operationId: string; action: ActionInfo };
```
There is no deprecated alias, ignored `deadline` field, compatibility overload, alternate timestamp option, or replacement pause flag in WP03.
The next drive package uses direct durable transitions. Deterministic tests gate commits and control hooks, providers, tools, and timers without adding production execution barriers.
## Host behavior
Hosts own execution budgets:
```text
invoke drive
→ terminal or durable waiting result: schedule normally
→ planned shutdown: stop routing/releasing work and close session processes
→ forced termination: replacement attaches and recovers durable open operations
```
A process-per-session host may stop routing work and close before exit, and use process termination as the hard fence for non-cooperative providers, tools, hooks, storage, or event listeners. This operational policy does not require a wall-clock field in `DriveOptions`.
WP03 does not add `stopAfterCheckpoint`, `pause`, `quiesce`, or in-process crash simulation. Those ideas remain outside the direct durable-drive design. In-process process-loss simulation is not a public core primitive: an old continuation requires fencing, for which process isolation is the reliable mechanism.
## Work
### Public types
In `packages/agent/src/harness/agent-harness.ts`:
- delete `DriveOptions.deadline`;
- delete the `DriveOutcome` `yielded` branch;
- preserve expected-id fencing, retry waiting, deferred waiting, and manual action branches unchanged.
No current runtime2 drive implementation exists, so this package adds no execution behavior or owner.
### Normative documentation
Update `packages/agent/docs/harness.md` completely:
- remove safe-yield scheduling language from non-goals/orientation;
- remove only the deadline half of cancellation/deadline prerequisite wording in §3.6, §5.6's `before_drive` row, and invariant 22; the cancellation prerequisite remains;
- remove deadline checks and yielded returns from the drive-pass pseudocode;
- remove deadline policy from pass joining, retry waiting, convenience composition, recovery, and public method prose;
- remove `deadline` from `DriveOptions` and `yielded` from `DriveOutcome`;
- remove deadline-specific event/turn requirements;
- remove deadline language from `before_drive` hook timing;
- remove deadline/yield requirements from future work rows;
- replace invariant 25 with the explicit no-wall-clock-policy invariant while retaining the rule that an admitted effect settles normally or is recovered after task loss;
- remove deadline races from the race catalog;
- update the drive-pass glossary.
Do not alter generic RPC timeout/deadline documentation or unrelated process/model-catalog timeouts. Those are invocation/transport policies, not `AgentLane.drive`.
Delete obsolete `packages/agent/docs/runtime2.md`; `harness.md` plus linked handoffs are the sole implementation plan and history needed for active work.
Mark WP02 complete in its handoff and Part 8. Add WP03 as the concrete cleanup package. Leave the former R2/R3 drive rows as historical future candidates, minus deadline/yield requirements, until the reviewed direct-drive handoff replaces them.
### Delete
- `packages/agent/docs/runtime2.md`
### Type tests
Extend `packages/agent/test/harness/types.test.ts`:
- assert `keyof DriveOptions` is exactly `"operationId" | "waitForRetry" | "pollDeferred"`;
- assert `DriveOutcome["kind"]` excludes `"yielded"`;
- add `@ts-expect-error` coverage proving callers cannot supply `deadline`;
- retain existing drive/result signatures.
Run the focused type test after editing it.
### Downstream compatibility
Search protocol, server, coding-agent, examples, and tests for structural mirrors or exhaustive `DriveOutcome` switches. Update only compile-time/type compatibility forced by the public removal. Coding-agent experimental worker/remote-runtime behavior and tests are out of scope.
The current protocol harness schemas expose prompt/run/watch DTOs, not `DriveOptions` or `DriveOutcome`; no protocol edit is expected unless the final search proves otherwise.
## Non-goals
WP03 does not implement or redesign:
- `drive`, `resume`, prompt conveniences, operation ownership, or latest-result lookup;
- manual actions or automatic barrier release;
- provider/tool execution, recovery, retry timers, deferred polling, abort, or terminal settlement;
- checkpoint pause/quiesce;
- close admission changes;
- worker/RPC cancellation or experimental remote prompting;
- storage/schema/migration behavior.
## Required checks
```bash
# No drive deadline/yield contract remains in active harness docs or source.
rg -n 'deadline|yield' \
packages/agent/src/harness \
packages/agent/test/harness \
packages/agent/docs/harness.md \
packages/agent/docs/work-packages
cd packages/agent
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run \
test/harness/types.test.ts
cd "$(git rev-parse --show-toplevel)"
git diff --check
npm run check
./test.sh
```
The removal grep may still match this handoff's historical problem explanation. Every remaining match must be reviewed; no active API, normative behavior, future acceptance criterion, or test expectation may retain the removed contract.
## Review
Before implementation:
1. Fable reviews this handoff against complete docs/source.
2. As explicitly requested by the user, `openai-codex/gpt-5.6-sol` reviews it with thinking level high.
3. Resolve all findings and repeat until no findings.
After implementation, repeat both reviews over the final documentation/type diff.
## Stop condition
Stop when:
- `DriveOptions` has no wall-clock budget;
- `DriveOutcome` has no `yielded` branch;
- active normative docs contain no deadline/yield behavior;
- future drive rows contain no hidden deadline requirements;
- type tests prove removal;
- unrelated timeout/deadline APIs remain untouched;
- focused tests, `npm run check`, and `./test.sh` pass;
- final Fable and Codex reviews have no findings.
Do not begin the direct durable-drive implementation in this package.

337
third_party/pi/packages/agent/docs/work-packages/04-mutation-publication.md vendored
View File

@@ -1,337 +0,0 @@
# WP04 — Mutation publication and event delivery
## Status
Complete. Phase A and the final implementation rereview passed Fable with no findings. Focused agent/server/SQLite tests, `npm run build`, `npm run check`, and `./test.sh` pass. `packages/agent/docs/harness.md` remains normative.
> Historical note: WP06 later replaced the lane-creation API and keyed line described in this completed handoff with atomic `AgentHarness.lane()` acquisition on one Session line. The event-publication guarantees remain current.
WP02 established atomic acceptance, recipient binding, and coherent lane watches. WP03 removed drive deadlines. WP04 removes the caller-operated event-delivery gate without weakening those guarantees and makes the historical `Session.createLane()` own lane creation end to end. The direct durable-drive package follows as WP05.
## Problem
A committing lane job currently uses a two-part event API:
```text
inside Session.mutate:
commit
publish process-local state
delivery = events.enqueue(batch, context)
outside Session.mutate:
await delivery.start()
```
The split preserves the correct boundary, but it is a footgun: calling `emit()` too early, calling `start()` too early, or dropping `start()` can violate observation semantics or stall the global event tail. The historical `Harness.createLane()` repeats the choreography manually.
Lane creation has a second one-off boundary. The historical `Session.createLane()` owns validation and the durable transaction, but Harness cannot publish its process-local `Lane` and bind `lane_created` recipients from that same commit continuation. Harness therefore opens `Session.mutate()` itself and calls exported `createLaneWithMutator()`.
WP04 removes both caller-operated seams while preserving current direct-listener and hook barriers.
## Required semantics
### Direct events remain awaited observations
Direct `events.on()` listeners remain passive but causally ordered:
```text
hook or preparation
→ commit
→ publish process-local state
→ bind and append event batch
→ release lane mutation line
→ await direct listeners
→ resolve operation
→ later hook or transition
```
Passive means a listener cannot transform the in-flight operation and listener failures are isolated as `handler_error`. It does not mean fire-and-forget. An extension may update process-local state in an event listener and inspect it from a later hook. Awaiting also supplies producer backpressure.
A direct listener must not call a state-mutating harness API: an emitted mutation would queue a later event behind the event currently awaiting that listener. Read-only lane calls remain legal.
Deliberate exceptions remain unchanged:
- watchers and RPC/watch consumers use their own buffered FIFO and are not awaited by operations;
- fault publication is fire-and-forget, and close never waits for listener completion;
- high-frequency tool updates enqueue every event and retain only the latest delivery promise; tool settlement awaits that promise before `after_tool` and outcome publication, which drains all earlier updates through global FIFO without per-update backpressure.
### Commit and recipient binding remain one continuation
Every event-producing committing lane job performs, in the exact continuation that observes successful commit:
```text
commit succeeds
→ publish complete process-local state
→ synchronously call emitBatch(batch, context)
- clone payloads
- bind current ordinary listeners and watchers
- append the complete batch to the global delivery tail
→ return from the mutation callback
```
There is no scheduler-owned after-release publication phase. Moving recipient binding into a later promise continuation creates a gap in which another task may observe committed state and register a listener before the old event binds recipients.
`emitBatch()` starts asynchronous delivery immediately and returns its completion promise. The mutation callback never awaits that promise. Listener code may begin after publication/binding but before the mutation line technically releases; a reentrant lane read queues behind the current job. The command carries the promise out of `Session.mutate()` and awaits it before resolving publicly.
This preserves the two legal watcher races:
```text
watcher registration first
→ snapshot-before + complete buffered batch
commit publication first
→ snapshot-after + no old event
```
### Ordering
- One non-empty `emitBatch()` call publishes one contiguous batch.
- Batches enter the existing global event tail in `emitBatch()` invocation order, including across lanes.
- Events within a batch retain source order.
- Direct listeners run serially in registration order.
- Same-lane committing jobs bind batches in lane mutation order.
- A lane procedure awaits each command's delivery before invoking its next hook or transition.
- Hooks on unrelated lanes may overlap; there is no total cross-lane hook order.
- Context remains the exact emitting invocation Context and is always the final parameter.
## Event bus contract
Replace the public/internal delivery split with:
```ts
class HarnessEventBus implements Events {
emit(event: HarnessEvent, context: Context): Promise<void>;
emitBatch(events: readonly HarnessEvent[], context: Context): Promise<void>;
}
```
`emitBatch()`:
1. returns an already-resolved promise for an empty batch or a closed bus;
2. synchronously structured-clones every payload and binds its current recipients;
3. appends one contiguous delivery to the existing global tail;
4. delivers cloned payloads serially with the emitting Context;
5. isolates listener failures and publishes non-recursive `handler_error` as today;
6. returns a promise that resolves after eligible direct listeners settle and never rejects because a listener failed.
Synchronous publication defects such as an uncloneable internal payload still throw in the caller's commit continuation and follow the existing harness-fault path.
Delete:
- `HarnessEventDelivery`;
- `HarnessEventBus.enqueue()`;
- caller-operated `start()`;
- delivery gates and `pendingStarts`;
- close-time forced gate release.
`close(error)` seals listener/watch registration and future publication immediately. Already appended batches retain their bound recipients and drain through the existing tail; listener completion still does not block Harness close.
## Lane command integration
The commit branch of `Lane.command()` returns a plain internal outcome containing the caller result and optional delivery promise:
```ts
const events = decision.events?.(commit) ?? [];
const delivery = events.length === 0
? undefined
: this.onEvent(events, context);
return {
kind: "return",
result,
...(delivery === undefined ? {} : { delivery }),
};
```
`onEvent` returns `Promise<void>` and delegates to `emitBatch()`. It is called only after commit, complete process-local state publication, and synchronous result materialization, as the final action before returning the mutation outcome.
After `Session.mutate()` returns:
```ts
if (outcome.kind === "reject") throw outcome.error;
await outcome.delivery;
return outcome.result;
```
Expected no-commit rejections publish no events. Commit/materialization/publication errors retain the existing harness-fault semantics.
## Session lane creation contract
The historical `Session.createLane()` owns validation, commit, and the synchronous committed-publication callback. Context remains last:
```ts
createLane(
name: string,
at: string | null,
configuration: LaneConfiguration,
onCommitted: ((context: Context) => void | Promise<void>) | undefined,
context: Context,
): Promise<SessionTree>;
```
The implementation performs:
```text
enter the prospective lane's mutation line
→ validate name, absence, complete lane shape, and target
→ commit lane configuration + leaf + idle lane state
→ synchronously invoke onCommitted(context) in that same commit continuation
→ retain its returned promise inside a non-thenable outcome object
→ return from the mutation callback and release the line
→ await the retained promise
→ return Session.view(name)
```
The callback receives neither `SessionTree` nor `SessionMutator`. It is only the process-local publication point. Its synchronous prefix must complete the publication needed before another same-lane job can run. Ordinary Session callers pass `undefined`.
If validation or commit fails, the callback is not invoked. If the callback throws after commit or its retained promise rejects after line release, the durable lane remains and the caller rejects; Harness converts that committed-publication defect to its existing fault path. A promise returned by the Harness callback is event delivery, whose listener failures are isolated by the bus.
The current lane validation/transaction implementation becomes private to Session. Delete exported `createLaneWithMutator()` and its direct tests.
Harness preconstructs a detached `Lane`, then calls Session:
```ts
const lane = this.buildLane(name, state);
await this.session.createLane(
name,
at,
this.seed,
(context) => {
if (this.closedError !== undefined) lane.seal(this.closedError);
this.lanesByName.set(name, lane);
return this.events.emitBatch(
[{ type: "lane_created", lane: name, at }],
context,
);
},
context,
);
return Result.ok(lane);
```
The callback's synchronous prefix publishes `lanesByName` and binds `lane_created` before the first creation job releases its line. A queued duplicate therefore cannot report `LaneExists` before the winner is visible through `harness.lane(name)`.
If close or fault wins while commit is admitted, the callback publishes the new Lane sealed. Emission on the already-closed bus is a resolved no-op, matching the existing admitted-creation race. The successful admitted creation still returns its sealed Lane.
## Scope
### Source
Modify:
- `packages/agent/src/harness/events.ts`;
- `packages/agent/src/harness/runtime2/lane.ts`;
- `packages/agent/src/harness/runtime2/harness.ts`;
- `packages/agent/src/harness/session/types.ts`;
- local Session implementations and tests required by the historical `createLane` signature;
- direct event primitive, acceptance, watch, lane, and harness tests.
Remote/experimental runtime behavior is not a design constraint for WP04. Session mutation authority remains process-local; do not add remote Session callback transport, protocol machinery, compatibility abstractions, or boundary tests.
### Documentation
Update normative `harness.md`:
- replace enqueue/start language with synchronous `emitBatch` binding and post-mutation awaiting;
- state accurately that listener execution may begin after publication/binding but before technical line release;
- retain awaited direct-listener, event/hook, watcher, Context, close, and ordering semantics;
- replace shared exported mutator-procedure lane creation with the historical `Session.createLane(onCommitted, context)`;
- update invariants, races, tests, glossary, and Part 8;
- link WP04 and move direct durable drive to WP05.
Update the historical WP02 handoff only where it points forward or claims the old mechanism remains current. Do not rewrite its completed-package record as though WP04 behavior landed in WP02.
## Non-goals
- fire-and-forget direct events or a public flush API;
- sequence/watermark delivery redesign;
- per-extension event queues;
- changing hook aggregation or event/hook causal barriers;
- changing watcher/RPC buffering;
- changing tool-update settlement barriers;
- making event-listener mutation safe;
- drive, breakpoints, providers, tools, recovery, retries, polling, abort, or terminal settlement;
- generic Session post-commit/after-release task APIs;
- remote callback execution or remote-runtime redesign.
## Required tests
### Event primitives
- `emitBatch` binds ordinary listeners and watchers synchronously;
- a listener registered after `emitBatch` but before delayed delivery receives nothing;
- a complete batch is contiguous and preserves event order;
- concurrent batch publication preserves invocation order;
- listener payload mutation remains isolated;
- listener rejection emits one non-recursive `handler_error` and does not reject delivery;
- empty and post-close batches resolve without delivery;
- already appended batches drain after close;
- no gate/start tests remain.
### Lane commands and watch
- direct listeners may perform reentrant read-only lane inspection without deadlock;
- a command does not resolve until its direct listeners finish;
- committed memory is visible to listeners;
- commit failure publishes nothing;
- a late listener that observes durable state receives no historical event;
- watcher-first yields snapshot-before plus the complete buffered batch;
- publication-first yields snapshot-after without replay;
- source Context identity survives delayed and buffered delivery.
### Lane creation
- Session callback runs exactly once after successful commit and never on validation/commit failure;
- callback synchronous publication occurs before a queued duplicate reports `LaneExists`;
- Harness Lane and durable configuration are visible to `lane_created` listeners;
- Harness waits for asynchronous `lane_created` listeners before resolving;
- admitted creation racing close publishes a sealed Lane and does not require event delivery;
- callback throw and retained-promise rejection after commit follow the documented committed-publication failure path;
- ordinary Session creation with `undefined` returns its view;
- no exported `createLaneWithMutator` remains.
### Ordering barriers
- acceptance events finish before `accept()` resolves;
- command resolution cannot overtake its awaited direct event delivery; WP05 tests the first subsequent procedure hook;
- event batches are globally ordered across concurrent lane publication;
- tool-update latest-delivery settlement behavior remains unchanged where implemented.
## Validation
After documentation:
```bash
git diff --check -- \
packages/agent/docs/harness.md \
packages/agent/docs/work-packages/02-atomic-run-acceptance.md \
packages/agent/docs/work-packages/04-mutation-publication.md \
packages/agent/docs/work-packages/05-direct-durable-drive.md
```
After implementation, run every modified focused test, then:
```bash
git diff --check
npm run check
./test.sh
```
Review the final implementation with Fable before declaring WP04 complete. Do not commit without explicit user approval.
## Stop condition
WP04 is complete when:
- event publication has one `emitBatch()` operation and no caller-operated gate;
- recipient binding remains in the exact commit-observation continuation;
- direct event delivery remains globally FIFO and awaited before public operation resolution;
- existing event/hook causal barriers remain intact;
- lane watches retain exactly the two coherent race outcomes;
- Session owns lane creation and invokes Harness publication before releasing the creation job;
- `createLaneWithMutator()` is gone;
- Context is trailing and source-identical throughout;
- focused tests, `npm run check`, and `./test.sh` pass;
- final Fable review reports no findings.

606
third_party/pi/packages/agent/docs/work-packages/05-direct-durable-drive.md vendored
View File

@@ -1,606 +0,0 @@
# WP05 — Direct durable drive
**Status: WP05 complete through M10: lane-owned inbox, immutable result records, 13 family-neutral leaves, atomic boundary planning, total cancellation/dispatch, public and replicated lane surfaces, documentation reconciliation, and lane-safe provider cache identity. Remaining assistant-output work is owned by the [mobile assistant-output handoff](../mobile-handoff/01-harness/05-assistant-output/message-update.md), outside the public-drive gate.**
WP06's Session/Branch/Lane separation is part of the foundation. Public drive is enabled; `watchSession` is the sole deferred Harness method.
Format 4 remains work in progress. Every durable type replacement in this package requires no migration and no compatibility decoder for the pre-redesign shapes.
The standalone-compaction inbox/promotion design previously in this document is **withdrawn** (§5). R1–R3 and M7/M8 replace it. M9 reconciled `docs/harness.md`, which is normative on its own again.
## 0. Mandatory reading
Read completely before implementation work:
1. `packages/agent/docs/harness.md` (normative; reconciled by M9)
2. `packages/agent/docs/runtime-simplification.md` (its 22-leaf list is replaced by R3)
3. `packages/agent/src/harness/session/types.ts`
4. `packages/agent/src/harness/session/values.ts`
5. `packages/agent/src/harness/runtime/lane.ts`
6. `packages/agent/src/harness/runtime/types.ts`
7. every existing `packages/agent/src/harness/runtime/drive/*.ts`
8. `packages/agent/src/harness/runtime/progress.ts`
9. `packages/agent/src/harness/runtime/restore.ts`
10. `packages/agent/src/harness/execution/{effect-gate,assistant,tools}.ts`
11. relevant focused runtime tests
Do not inspect Git history or removed runtime implementations. The current source and these documents are the only implementation inputs.
## 1. Runtime model (target)
### Lane authority
`Lane.state` is authoritative while a Harness owns its Session. It contains all orchestration state required to dispatch:
- tip;
- lane configuration;
- the lane inbox: tagged queued-input ids (R1);
- last operation id (R2);
- operation metadata;
- flat operation state, including control.
Every supported mutation commits on the one Session mutation line and publishes the matching `Lane.state` before releasing the line. Drive procedures never reread `laneState`, `operationMeta`, `operationState`, `branchTip`, `laneConfig`, or `operationResult` from storage.
`SessionReader` is used only to dereference content named by in-memory state and to enumerate operation-owned cleanup addresses:
- tree entries and branch context;
- pending entry payloads;
- assistant frame lists;
- tool arguments, checkpoints, and memos;
- structural preparations;
- staged tool outcomes;
- cleanup-prefix scans.
### One lane-owned inbox (R1)
Queued input is lane-owned, never operation-owned. `LaneState` (durable and process-local) carries one ordered inbox of `{ entryId, kind }` items with `kind: "steer" | "followUp" | "nextRun" | "write"`. Payload staging is unchanged: enqueue mints the entry id and writes `pendingEntry(id)`; the inbox holds ids only. Item lifecycle is admission → consumption | cancellation; terminal cleanup never touches queue payloads.
Tags are consumption-eligibility markers, not ownership. Enqueue always succeeds — during runs, structural operations, cancellation, and idle. Every drain point selects eligible items per tag, applying queue modes during selection, but places all selected items in the inbox's single global admission order. Tag grouping must never reorder user input. At acceptance, request prompt entries follow the selected inbox items because the request is the newest admission.
| Drain point | Eligibility and decision order |
| --- | --- |
| acceptance (idle lane) | write + nextRun (all), steer (`steeringMode`: all or oldest), and followUp (`followUpMode`) are eligible. Place selected items in global admission order, then request prompt entries. The acceptance transaction places them and removes their ids; `starting` drains nothing. followUp is eligible because an idle lane vacuously satisfies its "after current work" condition. |
| turn-end boundary pass (run) | write + steer are eligible before threshold/continuation planning and are placed in global admission order. followUp becomes eligible only at `may_finish`, before `before_run_end` and finish. nextRun is never eligible mid-run and never blocks finish. |
| idle direct append | queued write items are placed in admission order, then the new entry, in one commit |
| abort (M7) | steer + followUp are removed from the inbox, payload values deleted, and payloads returned; nextRun and write items stay |
**One decision, at most one commit (R1b).** A boundary pass may enter the mutation line several times for bounded reads and for `before_run_end` mediation, but it performs **at most one commit**, and that commit always lands in a state that neither drains nor rechecks (`assistant.ready`, `summary.deciding`, placed entries + `assistant.ready`, or a terminal transaction). No boundary decision ever commits back into `checkpoint`. A crash before the commit re-runs the whole decision with nothing consumed; a crash after it lands past the decision point. Consequently `skipInboxOnce` and `thresholdCheckedTriggerEntryId` are deleted: drains cannot re-fire because their target states do not drain, and the threshold marker is replaced by a guard derived from the branch itself — threshold fires only when `shouldCompact` holds **and** the branch's newest compaction entry is older than the trigger entry, so a committed threshold compaction is its own durable marker. Mode remainders stay queued and are consumed at later boundaries, giving the per-turn steer cadence and per-run-end followUp cadence.
There is no terminal drain. `cancelQueued` triage is unchanged and single-location. Queued input survives operation termination (including abort, for nextRun/write) until consumed or cancelled; this is a deliberate product decision.
### Flat state (R3)
`OperationState` has one `at` discriminator with **13** direct leaves:
- `starting`
- `checkpoint`
- `assistant.ready`
- `assistant.effect_pending`
- `assistant.retry_wait`
- `tools`
- `deferred.suspended`
- `deferred.effect_pending`
- `summary.deciding`
- `summary.ready`
- `summary.effect_pending`
- `summary.retry_wait`
- `navigation.ready_to_commit`
Every leaf carries one uniform scope `{ control, settings, latestAssistantEntryId }`. There are no `run.`/`compaction.`/`navigation.` leaf prefixes, no per-family scope types, and no intersection zoo. The four `summary.*` leaves carry a `SummaryTask` whose `boundary` datum (a closed three-arm union, §8) decides what happens at the result boundary. `starting` and `navigation.ready_to_commit` are intent-locked entry leaves. `ToolBatch`/`ToolCall` remain the nested child collection state machine. `Control` stays orthogonal and loses its drained fields when M7 installs drain-and-return abort.
### Durable operation results (R2)
Every terminal transaction writes one small immutable result record at `operationResult(operationId)` (namespace `pi.result`, operation id key). The record is lane-lived; keeping it outside `pi.op.*` keeps that namespace's "deleted no later than the terminal transaction" grammar total and exception-free. `LaneState.lastOperationId` points at the newest record. `laneLastResult` (value, address constructor, and type union) is deleted.
- `drive(id)` becomes total for settled operations: current id → install/join; record exists → return the record itself; neither → `OperationMismatch`.
- Recovery and attachment never read result records; they are observation only.
- Terminal-control invariant: a terminal transaction that executes under durable `cancel_requested` always records `status: "aborted"`; equivalently, any other status implies running terminal control. (Every non-aborted terminal path goes through `continueOperation`, which diverts under cancellation, and the mutation line serializes the marker against the terminal commit.)
- There is no listing, filtering, pagination, or retention API for records — and no hydration layer. `getResult(operationId)` and the drive settled arm are the entire read surface; both return the record with no entry dereference, so observation can never fault on missing entries.
### One lane-owned Drive
Unchanged: one process-local Drive pass per lane; first matching caller installs, later matching callers observe the same completion; no caller owns it after installation; invocation cancellation rejects only that caller's observation; an installed Drive is never abandoned or replaced in-process; `requestAbort(operationId)` is the only durable operation cancellation; a stale operation id returns `OperationMismatch` and affects nothing. `Drive.context` removes the installing invocation signal.
### Close
Unchanged: close is not abort. It seals mutation admission, rejects local observations with `HarnessClosed`, observes detached pass failures, drains mutations admitted before the seal, and closes the Session. It writes no cancellation marker or synthetic terminal state and never replaces a Drive.
### No external finalization
Unchanged: no live-process external-finalization path, `OperationEnded`, `finalizedOutcome`, ownership-loss result, or exact-Drive ABA fence.
## 2. Procedure shape
Live procedures are ordinary straight-line async functions:
```text
prepare
→ commit intent
→ admit and await effect
→ commit settlement
```
A procedure is the sole writer that changes its top-level `at` leaf. `requestAbort` changes only `control` (and the lane inbox); inbox admission changes only the lane inbox. Operation state has exactly two supported writers: the procedure and `requestAbort`.
The Lane supplies `continueOperation` and `settleOperation` as today. A terminal decision appends the universal suffix:
1. procedure-specific publication and cleanup writes;
2. `setValue(operationResult(operationId), record)`;
3. idle `laneState` with `currentOperationId: null`, `lastOperationId: operationId`, and the preserved inbox;
4. idle process-local projection;
5. terminal result and event materialization.
The state argument is a type capability established by dispatcher control flow, never compared with current state at runtime.
## 3. Remaining concurrency checks
Keep checks only where another supported writer can change the relevant fact:
1. `requestAbort` versus effect admission and settlement;
2. lane-inbox arrival before a drain point, a summary result boundary, or terminal finish;
3. parallel tool-call statuses and source-ready placement;
4. queued frame/checkpoint writes versus settlement;
5. invocation memo/checkpoint calls versus effect completion;
6. retry timers versus cancellation/close;
7. deferred permit consumption;
8. accept/claim serialization;
9. external/provider/content validation.
Do not reintroduce operation identity, expected-`at`, exact-Drive, or ownership-loss checks into ordinary transitions.
`requestAbort` keeps the two-step gate order: `beginAbort` before the cancellation mutation, commit `cancel_requested`, `signalAbort` after the commit.
## 4. Completed milestones
### M0 — Withdrawn execution-step controls
Complete. Breakpoints, manual drive, and drive deadlines are absent.
### M1–M2 — Foundations and terminal mechanics
Complete. Includes the split effect gate, deterministic gated storage, authoritative process-local Lane projection, commit-result usage totals, progress channels, terminal cleanup, and immutable operation-result observation.
### M3 — Assistant generation and recovery
Complete. Includes `run.starting → run.checkpoint`, assistant ready/intent/effect/settlement, frame persistence and cleanup, configuration failure, unknown-outcome assistant recovery, and response/usage/state atomicity.
### M4 — Retry and deferred
Complete. Includes durable retry waits, local wait policy, one deferred poll permit per pass, unknown-poll replacement under fresh ids, and event-stream-preserving `Models.streamDeferred`.
### M5 — Durable tools
Complete. Includes planned/effect-pending/outcome-ready/completed calls, safe replay, unsafe interruption, invocation memos, bounded checkpoints, completion-order staging, source-order placement, sequential and parallel modes, and tool-reported usage.
### Pre-M6 simplification
Complete. Canonical flat `at` types, `continueOperation`/`settleOperation`, removal of the installer-owned Drive model, and conversion of M3–M5 procedures.
### M6 structural foundation
Committed (`9b23c6583`): `runtime/drive/structural.ts`, `test/harness/runtime/drive-structural.test.ts`, the one-provider-request seams in the compaction modules, threshold routing, overflow preparation, per-request structural intent/usage settlement, hook decisions, structural retry/recovery, and unsummarized navigation commit. Its family cross-product layer (triplicated quadruples, three-armed publishers) is rebuilt by R3; its effect seam, preparation plumbing, threshold/overflow logic, and navigation commit are retained.
## 5. Withdrawn: standalone-compaction inbox and run-continuation promotion
The design that gave `compaction.*` leaves a `RunScope` inbox and a one-way cross-family move into `run.*` at the result boundary is withdrawn. Its problems: a relaxed intent/state invariant with bespoke restore rules, `compact()` resolving with a run outcome that does not fit `CompactionResult`, suppressed `run_start` producing unbalanced event brackets, fused-consumption rules to avoid `cancelQueued` stranding, and promotion-specific arms in reconciliation, cleanup, watch, and admission.
Replacement, spread across this package:
- queued input during any operation is lane-owned (R1) — no operation inbox exists to promote;
- continuation after standalone compaction/navigation is a **second ordinary run operation** with a fresh id, composed in the convenience layer (M8);
- each operation delivers exactly one result; both results are returned by the convenience call and both are durable records (R2).
No code implements promotion. M9 removed the withdrawn design from `harness.md`; this section remains only as the historical decision record.
## 6. R1 — Lane-owned tagged inbox
### Goal
Move all queued input to one lane-owned tagged inbox first (R1a, implemented), then — after R3 — replace the transitional stepwise checkpoint drains with one shared atomic boundary planner (R1b). `Control.drained*` remains until M7 introduces drain-and-return abort; it is not part of inbox ownership.
### R1a — Ownership relocation
- `src/harness/session/types.ts` — durable `LaneState { currentOperationId, inbox }` with `InboxItem { entryId, kind }` replaces `pendingNextRun`; delete `Inbox` and `RunScope.inbox`. Keep `Control.drained*`, `skipInboxOnce`, and `thresholdCheckedTriggerEntryId` temporarily.
- `src/harness/runtime/types.ts` — process-local `LaneState.inbox` replaces `pendingNextRun`; `LanePatch.inbox` pairs durable lane-state replacement with process-local publication.
- `src/harness/runtime/lane.ts` — acceptance selects per tag and mode but places selected items in global admission order, then request prompt entries; it deletes selected pending values and removes their ids atomically with the `LaneBusy` check. A lone queued write never validates acceptance. `append` during any operation enqueues a write-tagged lane item without touching operation state; idle `append` places queued writes first, then the new entry, in one commit. `watch` derives queues and pending writes from the lane inbox.
- `src/harness/runtime/drive/checkpoint.ts` — transitional stepwise drains read and replace the lane inbox rather than operation state. `skipInboxOnce` and `thresholdCheckedTriggerEntryId` continue to protect those temporary multi-commit boundaries.
- `src/harness/runtime/drive/terminal.ts` — operation cleanup never deletes lane-inbox payloads. Staged `outcome_ready` results, drained abort payloads, and other operation-owned addresses are unchanged.
- `src/harness/runtime/drive/{generation,response,deferred,tools,structural}.ts` — `runScopeOf` and scope copies drop `inbox`; ordinary operation-state transitions cannot clobber concurrent input.
- `src/harness/runtime/restore.ts`, fork/legacy normalization, and focused tests use the tagged lane inbox.
### R1b — Shared atomic boundary planner
**Lands after R3**, because pre-R3 the structural result boundary is spread across the triplicated publishers: fusing the planner into three arms and then collapsing them in R3 would be double work, while post-R3 the single `ResultBoundary` switch is exactly one planner call site. The two `CheckpointData` patch fields ride through R3's mechanical rename as inert baggage.
**Core invariant: no boundary decision ever commits back into `checkpoint`.** `checkpoint` is the durable "boundary pending" resting leaf, entered only from settlements, run-start consumption, and deferred redemption. Every exit of its pass is `assistant.ready` (with any placed entries), `summary.deciding`, a placed follow-up + `assistant.ready`, or the terminal transaction.
One shared boundary planner with a process-local `threshold: "check" | "skip"` parameter has exactly two callers:
- the `checkpoint` pass (`threshold: "check"`): select write + steer (modes, global admission order) → threshold guard → continuation → followUp (at `may_finish`) → `before_run_end` → finish; one commit;
- the structural result boundary for `resume_checkpoint` (`threshold: "skip"`): the **same planner runs inside the publication commit** — compaction entry (on success) first, then selected write/steer items, then the routed continuation, all in one transaction. Any placed conversational item routes to `assistant.ready` with that item as trigger, overriding a `may_finish` resume continuation (steer semantics). With nothing queued: `need_assistant` resumes commit `assistant.ready` directly; `may_finish` resumes commit `checkpoint{may_finish}` as the resting leaf and the live pass continues into the finish phases.
**Threshold without the durable marker.** The guard is derived from the branch: threshold fires only when `shouldCompact` holds and the newest compaction entry is older than the trigger entry. A committed threshold compaction is therefore its own marker — any crash re-entry sees the newer compaction and skips. A **declined** threshold compaction publishes no entry and needs no marker: decline never lands on `checkpoint`. The decline decision stays process-local at `summary.deciding` until its single continuation commit (`assistant.ready`, follow-up + ready, or the terminal after `before_run_end`); a crash in that window restores `deciding` and re-runs `before_compaction` under its documented repetition contract. A live decline/recheck loop is structurally impossible because no decline path re-enters the threshold-checking pass.
**`before_run_end` without hooks on the mutation line.** The finish arm is the one multi-phase case: (1) on-line verdict, no commit — reached only at `may_finish` with no eligible write/steer/followUp; (2) hook runs off the line; (3) on-line **replan from current state** — if the inbox gained eligible items or control changed, the stale hook result is dropped and the planner takes the new decision; otherwise the single commit is the follow-up injection (entry + `assistant.ready`) or the terminal transaction. Stale-dropping is replanning, not a version check.
Concrete control trace (post-R3 names; threshold compaction with `may_finish` resume and a steer arriving mid-compaction):
```text
TX1 settlement: response n7 + usage + tip, S=checkpoint{may_finish, trigger n7}
pass (check): shouldCompact && newestCompaction < n7 → prepare
TX2: preparation, S=summary.deciding{boundary: resume_checkpoint{may_finish, n7}}
TX3 (any time): steer s1 → pendingEntry(s1), laneState.inbox += s1
TX4..n: deciding → ready → effect_pending → nested request/usage settlements
TX-pub (skip): insert compaction c1, tip=c1, insert s1 entry (parent c1),
delete pendingEntry(s1), laneState.inbox -= s1,
S=assistant.ready{trigger s1} ← one commit, checkpoint never re-entered
…turn answers s1; next settlement → checkpoint{may_finish, trigger n9}
pass (check): newestCompaction c1 > … tokens small → no threshold; inbox empty; verdict finish
→ before_run_end off-line → replan: still empty, still running
TX-final: record + cleanup + laneState{current: null, last: op}
```
Crash between TX4..n and TX-pub → structural recovery (attempt unknown, per existing rules); crash during the hook window → `checkpoint{may_finish}` restored, pass re-runs (threshold skipped by the recency guard), hook re-runs per contract.
Slices, in order (the order is load-bearing — deleting the marker while decline still re-entered `checkpoint` would live-loop):
1. **R1b-1**: fuse the future shared planner into the structural `resume_checkpoint` boundary (`threshold: "skip"`); structural success/decline route directly except for a `may_finish` resting checkpoint, which remains safe for either outcome while the durable marker still exists. Keep the route inline until R1b-2 creates its second caller. Both patch fields are still written and checked by the old checkpoint pass — harmless.
2. **R1b-2**: convert the `checkpoint` pass to single-commit exits with the recency guard, implement the three-phase finish arm, delete `skipInboxOnce` and `thresholdCheckedTriggerEntryId`.
### Rules
- The §1 eligibility table, global placement order, and one-decision-one-commit target are normative.
- Queue modes apply at selection time. Remainders preserve global admission order and are consumed only at later eligible boundaries.
- nextRun items are never consumed mid-run and never block run finish.
- Deferred-suspension enqueue always admits; consumption happens at the post-redemption boundary.
### Focused validation
Enqueue during idle/run/structural/cancelled states always admits; acceptance preserves global order with modes applied and remainders retained; `one-at-a-time` cadence gives each remaining steer its own later boundary; followUp is captured at idle acceptance and only at `may_finish` during a run; a boundary crash before its commit consumes nothing; threshold is checked at most once per boundary by construction; idle append places queued writes first; items survive every terminal status and process loss; `cancelQueued` works at every boundary; watch groups by tag without changing order; no terminal transaction deletes a lane-inbox payload; a lone queued write never validates acceptance.
## 7. R2 — Neutral operation outcome and durable result records
### Goal
Replace the three per-family outcome unions and `laneLastResult` with one immutable per-operation result record. The record **is** the public settled outcome; there is no separate outcome type, no embedded entries, and no hydration.
### Types
```ts
type TerminalStatus = "completed" | "declined" | "aborted" | "failed";
/** Stored at operationResult(operationId) — namespace "pi.result" — by the terminal transaction. Immutable, lane-lived. */
interface OperationResultRecord {
operationId: string;
kind: "run" | "compaction" | "navigation"; // meta.intent.kind; matches OperationAdmission.kind
status: TerminalStatus;
error?: OperationError; // status "failed"
fromTipId: string | null; // meta.sourceTipId — start of the transcript segment
tipId: string | null; // lane tip at terminal — end of the segment
startedAt: number; // Unix ms, from meta
endedAt: number; // Unix ms, Date.now() at terminal planning
}
/** Convenience-only suspension observation for prompt()/resume() (M8). Never stored. */
interface SuspendedRun { operationId: string; status: "suspended"; deferred: DeferredHandle }
```
The record is a disposition plus a pointer to the transcript segment `(fromTipId, tipId]`. It never lists intermediate work (compactions, turns, tools) and never embeds entries. There is **no `tipEntry` and no `OperationOutcome` union**: a caller that wants the payload dereferences `tipId` with the already-public `getEntry`/`findEntry` (for runs, the final message was additionally delivered by `message_end`/`entry_added`). This buys three things: terminal planners perform zero outcome reads, observation cannot fault on a missing entry (a `getEntries` throw inside hydration was a harness-fault path for a read-only convenience), and the future protocol never serializes `Entry` inside results — a result frame is eight flat fields. `SuspendedRun` is the one non-terminal observation, exists only on convenience returns, and carries nothing derivable elsewhere.
### Files
- `src/harness/session/types.ts` — add the record; delete the `LaneLastResult` union; `LaneState` gains `lastOperationId: string | null`; delete `failure_drain`, `RunFailureDrainOperation`, and `FailureProvenance`.
- `src/harness/session/values.ts` — `operationResult(operationId) = value<OperationResultRecord>("pi.result", operationId)`; delete `laneLastResult`. The `pi.op.*` lifetime grammar stays total: no operation-lived namespace survives its terminal transaction.
- `src/harness/agent-harness.ts` — delete `RunOutcome`, `CompactionOutcome`, `NavigationOutcome`, `OptionalFinalAssistant`, `TerminalOperationOutcome`, `ResumeOutcome`, and the transitional `OperationOutcome` union; `DriveOutcome.settled` carries `OperationResultRecord` directly (no duplicate `operationId` field); result aliases become
`RunResult = Result<OperationResultRecord | SuspendedRun, …>`,
`CompactionResult = Result<{ compaction: OperationResultRecord; run?: OperationResultRecord | SuspendedRun }, …>`,
`NavigationResult = Result<{ navigation: OperationResultRecord; run?: OperationResultRecord | SuspendedRun }, …>`,
`ResumeResult = Result<OperationResultRecord | SuspendedRun, …>`;
`QueueResult` loses `NoActiveRun` and absorbs `NextRunResult`. `SuspendedRun` is declared here but constructed only by M8 convenience paths.
- End-event payloads, one shape per event type: `run_end` and `navigation_end` carry `{ runId, status, error?, fromTipId, tipId }`. `compaction_end` is a segment event, not a terminal event — it also closes in-run threshold/overflow brackets — and always carries `{ runId, reason, status: "completed" | "declined" | "failed" | "aborted", error?, entryId? }`, where `entryId` names the compaction entry on success. Embedded `entry`/`summaryEntry`/final-assistant fields are dropped everywhere (`entry_added`/`message_end` already delivered payloads).
- `src/harness/runtime/types.ts` — `FinishDecision` carries the record instead of `lastResult`.
- `src/harness/runtime/lane.ts` — terminal suffix per §2; `getLastResult` is replaced by `getResult(operationId, context): Promise<OperationResultRecord | undefined>` — one `getValue`, nothing else; `inspectExecution` reports `lastOperationId` only.
- `src/harness/runtime/drive/terminal.ts` — `hydrateTerminalOutcome`/`hydrateOperationOutcome` are deleted outright; the file keeps only `operationCleanupWrites` and the `operationResultRecord` constructor. Finish decisions materialize `{ kind: "settled", outcome: record }` from the record they already built — no reader dereference inside any terminal planner.
- `src/harness/runtime/drive/{checkpoint,response,structural,recovery,deferred,tools}.ts` — every finish decision constructs the record; `runCompletion`, per-family last-result construction, and final-assistant plumbing in results are deleted. Every former `failure_drain` producer instead commits terminal failure directly: response publication and cleanup stay in the same transaction, in-run structural failure closes `compaction_end` then `run_end`, and queued lane input remains untouched for a later ordinary run. (`latestAssistantEntryId` stays in scope for cancellation classification and checkpoint logic; it just no longer feeds results.)
- `src/harness/runtime/restore.ts` — restore `lastOperationId`; attachment never reads records.
- Focused tests.
### Rules
- Records are written exactly once, by the terminal transaction, and never deleted, updated, or read by recovery.
- Terminal-control invariant (§1): a terminal transaction under durable `cancel_requested` records `aborted`; no path commits any other status under cancellation. M7 adds the enforcing test; M9 adds the Part 9 invariant.
- Forks exclude `pi.result` values.
- `drive(id)` arms: current → install/join; record → returned directly; neither → `OperationMismatch`.
### Focused validation
Record written for every terminal path of every operation kind and status; `drive(id)` returns records for arbitrarily old ids across reopen; `getResult` is a plain value read returning the record or `undefined`; no terminal planner or observation path reads entries for the result; `lastOperationId` restore; suspension never stored; fork exclusion; segment pointers correct (`fromTipId` = pre-acceptance tip, including navigation); `compaction_end` closes both standalone and in-run brackets, including `aborted` via reconciliation; each former failure-drain producer finishes in one transaction and preserves every lane-inbox item for a later run.
## 8. R3 — Family-neutral leaves and structural rebuild
### Goal
Collapse 22 leaves to the 13 in §1; rebuild `structural.ts`'s state-shape layer on one summary quadruple with an explicit result-boundary datum.
### Types
```ts
interface OperationScope { control: Control; settings: RunSettings; latestAssistantEntryId: string | null }
type ResultBoundary = // closed; do not extend
| { kind: "resume_checkpoint"; resumeAfter: CheckpointData } // in-run threshold/overflow
| { kind: "finish" } // standalone compaction
| { kind: "commit_navigation"; targetId: string; label?: string };
interface SummaryTask {
taskId: string;
reason?: "manual" | "threshold" | "overflow";
customInstructions?: string;
boundary: ResultBoundary;
}
```
The summary algorithm kind is **derived from the boundary** (`resume_checkpoint`/`finish` → compaction; `commit_navigation` → branch summary); it is not stored, so no contradictory kind/boundary combination is representable. `summary.ready/effect_pending/retry_wait` additionally carry the generation snapshot (result entry id, configuration, stream options, retry policy, attempt counters); the snapshot drops the `taskId`/`kind`/`reason` fields it previously duplicated. Canonical declarations land in `session/types.ts`. Uniform scope means compaction/navigation acceptance captures `settings` exactly like run acceptance; `latestAssistantEntryId` stays null for structural intents.
### Reachability (restore check; replaces the intent-prefix check)
| Intent | Admissible leaves |
| --- | --- |
| run | `starting`, `checkpoint`, `assistant.*`, `tools`, `deferred.*`, `summary.*` with boundary `resume_checkpoint` |
| compaction | `summary.*` with boundary `finish` |
| navigation | `navigation.ready_to_commit`; `summary.*` with boundary `commit_navigation` |
Forbidden and unreachable by construction (assert in tests, never implement): any edge into `starting` or `navigation.ready_to_commit` other than acceptance; `summary.*` → `tools`/`assistant.*` directly; `navigation.ready_to_commit` → `summary.*`; terminal → anything.
### Result-boundary rule
Every summary boundary (hook decline, hook-supplied result, generated success, terminal generation failure, model unavailability) plans on the mutation line and switches on `boundary` in one visible place:
- `resume_checkpoint` → publish result and restore the marked checkpoint on success or threshold decline; overflow decline or structural failure closes the compaction bracket and terminal-fails the run in the same commit;
- `finish` → publish the compaction entry and terminal-complete (or terminal declined/failed) in one commit;
- `commit_navigation` → the single move/summary/label/terminal commit (or terminal declined/failed).
A `cancel_requested` boundary defers to reconciliation and never takes its continuation.
### Files
- `src/harness/session/types.ts` — the 13-leaf union, `OperationScope`, `SummaryTask`, `ResultBoundary`; delete `RunScope`, `CompactionScope`, `NavigationScope`, `NavigationSummaryScope`, `RunCompactionScope`, `StructuralTask`, per-family `Extract` aliases, `isRunOperationState`.
- `src/harness/runtime/drive/structural.ts` — rebuild: keep `durableCompactionPreparation`/readers, the nested request seam, `performStructuralAttempt`, `runCompactionThreshold`, `prepareOverflowCompaction`, `commitNavigation`; delete the seven union aliases, the `startsWith` guards, and the three-armed `effectPendingFromReady`/`retryWaitFromEffect`/`readyFromRetryWait`/`publishStructuralReady`/`publishStructuralDecline`/`publishStructuralFailure`/`publishCompactionResult`-vs-`publishNavigationSummary` split; one quadruple of converters plus one boundary switch replace them.
- `src/harness/runtime/drive/{checkpoint,generation,response,recovery,deferred,tools,tool-placement,terminal}.ts` — leaf literal renames (`run.checkpoint` → `checkpoint`, …); `response.ts` overflow arm constructs `summary.deciding` with `resume_checkpoint`; no other behavior change.
- `src/harness/runtime/progress.ts` — frame/checkpoint fences currently match the leaf literals `"run.assistant.effect_pending"`, `"run.deferred.effect_pending"`, and `"run.tools"`; retarget them to the neutral leaves.
- `src/harness/runtime/restore.ts` — reachability predicate above.
- `src/harness/runtime/lane.ts` — `capturedModel` and `watch` switch over the neutral leaves; acceptance writes neutral initial leaves.
- `docs/runtime-simplification.md` — replace the 22-leaf list and status.
- `test/harness/runtime/drive-structural.test.ts` and other focused tests — retarget; behavioral assertions survive.
### Focused validation
Everything the M6 foundation covered, re-expressed over neutral leaves, plus: reachability accept/reject matrix (including corrupted boundary/intent combinations faulting restore); each boundary arm × {success, hook result, decline, failure, model absence} × {running, cancelled}; a post-terminal leak scan asserting every `pi.op.*` address is gone for every leaf while the `pi.result` record is present; crash/reopen at every leaf under every intent that can reach it.
## 9. M7 — Cancellation reconciliation and total switch
### Goal
**Implemented and reviewed.** Make the internal graph total without public wiring.
Create `src/harness/runtime/drive/reconcile.ts`, `src/harness/runtime/drive.ts`, and focused reconciliation and switch tests.
### `requestOperationAbort`
Package-private expected-id primitive:
1. reject with `OperationMismatch` when the expected id is not the current durable operation, including when that id already has a settled result record;
2. synchronously `beginAbort` on a matching live Drive;
3. one commit on the Session line: `control = cancel_requested` (with no drained fields) **plus** removal of steer/followUp-tagged ids from the lane inbox and deletion of their `pendingEntry` values; payloads are read in the same mutation and returned; nextRun/write items stay;
4. `signalAbort` after the commit;
5. after a newly requested abort commits, publish `{ type: "operation_abort", operationId, steer, followUp }` and return once cancellation is durable. This family-neutral event replaces `run_abort`/`runId`; the lane snapshot already identifies the operation kind. Repeat calls against the same current cancelled operation publish nothing, drain nothing further, and report `newlyRequested: false`.
With no Drive but a matching current durable operation, it commits the same marker and starts no pass. There is no `control.drained*`; the drained payloads exist only in the returned result (accepted loss: a crash inside the abort window loses their content).
### Reconciliation
Checked before `before_drive` and ordinary dispatch; a single switch over the 13 leaves; never starts new ordinary work. It must handle:
- assistant and deferred effects with live or reconstructed results;
- planned/effect-pending/outcome-ready tool calls;
- structural process-local results (discarded unless already atomically published);
- cancelled summary boundaries finishing `aborted` without taking their `ResultBoundary` continuation;
- retry waits, checkpoints, and suspended deferred work;
- best-effort deferred-provider cancellation using the Drive's close-only signal, never its already-triggered operation-abort gate;
- the aborted terminal transaction (result record `status: "aborted"`);
- the terminal-control invariant test: no terminal path commits a non-aborted status under cancelled control (this keeps §10's continuation rule decidable from the record alone).
Queued lane-inbox items are **not** applied or deleted by reconciliation; nextRun/write items simply remain queued.
Each Drive owns a private close controller and exposes its signal to mandatory cleanup. `closeGate()` aborts that controller on harness close or fault; operation abort does not. Deferred-provider cancellation uses this close-only signal, so the cleanup request can start after durable operation cancellation but cannot outlive harness shutdown. Do not add a harness-global cancellation controller.
### Total switch
`runtime/drive.ts` owns one direct `state.at` switch over 13 leaves. It imports only complete procedure modules. No graph table, action interpreter, ownership-loss arm, external-finalization arm, or storage-state reload. A `continue` result must correspond to a replaced `Lane.state` projection **or** to currently observable `cancel_requested` control, which the switch routes to reconciliation before ordinary dispatch; any other unchanged continuation is an invariant defect.
Public methods remain guarded through M7.
## 10. M8 — Public surfaces
**Implemented.** Execution guards were removed only after every leaf and reconciliation path became total.
Order:
1. admit compaction/navigation requests (acceptance captures `settings`; writes the R3 entry leaves);
2. implement `drive` install/join/record lookup (three arms, §7);
3. expose `requestAbort`;
4. add convenience compositions;
5. add queues/configuration/usage/idle surfaces;
6. retain `watchSession` as the sole `SliceNotImplemented` method.
### Queues
`steer`/`followUp`/`nextRun` are tag sugar over one enqueue and always admit. `queue_update` and `LaneSnapshot.queues` expose the same tagged ordered inbox, including pending writes; clients group by tag without reordering it.
### Client replication surface
Findings from a working RPC-shaped client replica (mini TUI exercise) fold in here. The goal is one testable contract: a remote client renders exclusively from a replicated `LaneSnapshot` and the event stream, with no side-channel getters.
- **Normative reducer.** Export `reduceLaneSnapshot(snapshot, event): LaneSnapshot | { rebase: true }` from `packages/agent`. It is the single supported event fold: in-run `compaction_start`/`compaction_end` are segments and must not clear `operation` (the fold knows the open operation's kind, so no event field is needed); `run_suspend` keeps `operation` non-null with `deferred` set; only `run_end`, `navigation_end`, and `compaction_end` under a compaction-kind operation are operation-terminal. `navigation_end` returns `{ rebase: true }` because the moved tip may lie outside the replica. Conformance assertion: for every non-navigation flow, folding a `watch()` snapshot over its own event stream equals a later `watch()` snapshot — this equivalence test is what keeps the event vocabulary complete.
- **Snapshot completeness.** `LaneSnapshot` gains `configuration: LaneConfiguration`, `lastResult?: OperationResultRecord` (replacing the bare `lastOperationId` field), a real `faulted` flag (the current hardcoded `false` is a defect), and session `stats: SessionStats` as the usage baseline (usage events already carry `totals`; the snapshot supplies the value before the first event).
- **Replicable configuration events.** Every `config_update` variant whose value is data carries `value`/`previous` (`streamOptions`, `retryPolicy`, `compactionSettings`, `steeringMode`, `followUpMode`, plus the existing lane variants); `tools`/`resources` remain notification-only because registries are code — clients re-fetch names.
- **Identity-based `setModel`.** `setModel` accepts `ModelIdentity` (`{ provider, modelId }`); a live `Model` object is a process-local registry concern, and an unregistered identity fails in-band at generation exactly like any registry absence.
- **Re-basing.** `WatchHandle.resnapshot(context): Promise<LaneSnapshot>` captures a fresh snapshot on the mutation line, serialized against the same stream — the recovery path after `{ rebase: true }` with no teardown/re-subscribe choreography.
- **Start-event timestamps.** `run_start`/`compaction_start`/`navigation_start` carry `startedAt` (operation starts: `meta.startedAt`; in-run segment starts: the commit timestamp), so folds never invent times.
### Convenience compositions and continuation
- `prompt`/`skill`/`promptFromTemplate`: accept + drive; return `Result<OperationResultRecord | SuspendedRun, …>` — the record for terminal outcomes, `SuspendedRun` when the run defers.
- `compact`/`navigateTree`: accept A + drive A. If A settled with status `completed`, `declined`, or `failed` and eligible steer/followUp/nextRun items exist, accept a continuation run B with the **public empty-prompt request** (`{ kind: "prompt", prompt: "" }`): acceptance places no request messages and is legal exactly when its capture places at least one conversational item; `OperationMeta.intent` is an ordinary run intent with empty `promptEntryIds`. Then drive B and return `{ compaction|navigation: A, run?: B }`. Never after `aborted`: abort already drained steer/followUp, and the terminal-control invariant (§7) makes `status ∈ {completed, declined, failed}` imply the operation was never durably cancelled, so the rule is decidable from the record alone. B is an ordinary run — it emits `run_start`, runs `before_run` (with `prompt: []`, the existing captured-only acceptance shape), and owns the full run graph with no continuation-aware special case.
- Continuation races: a competing accept that wins the idle window captures the queued input into its own run; the continuation accept then fails empty (`InvalidMessage`) or `LaneBusy`, and the convenience returns A without `run`. Both histories are valid. A crash between A's terminal and B's acceptance leaves the items queued; no auto-start on reopen.
- `resume`: inspect + drive current id, one deferred-poll permit; returns `OperationResultRecord | SuspendedRun`.
- `abort`: inspect + `requestAbort` + ensure a reconciliation pass; returns the drained steer/followUp payloads.
- `getResult(operationId)`: public.
- Equivalence: every convenience call ≡ its primitive composition using only public request kinds (`compact()` with continuation ≡ `accept(A); drive(A); accept({ kind: "prompt", prompt: "" }); drive(B)`), byte-identical writes and events — externally reproducible by any scheduler.
### Focused validation
One install and same-id joins; stale-id isolation; record lookup for old ids; caller cancellation before/after installation; close/fault during cooperative and non-cooperative effects; accept/drive vs convenience equivalence including continuation via the public empty-prompt request; steer-only, followUp-only, and nextRun-only continuation each start run B; both continuation race orders; abort drain-and-return including the both-orders abort/settlement race; enqueue during every state through public surfaces; full crash matrix across all 13 leaves; no unhandled detached rejection; no `SliceNotImplemented` except `watchSession`; reducer fold-equivalence across every non-navigation flow (including in-run compaction segments, suspend/resume, retry waits, and reattach mid-stream); `navigation_end` fold returns rebase and `resnapshot` restores equivalence; snapshot `configuration`/`lastResult`/`faulted`/`stats` replicate through events alone.
## 11. M9 — Documentation reconciliation
**Implemented.** `docs/harness.md` is reconciled to the runtime and normative on its own. Updated sections include:
- §1.3 address table and lifetime grammar: new lane-lived `pi.result` namespace, `laneLastResult` removed, `LaneState.inbox`/`lastOperationId`; `pi.op.*` stays strictly operation-lived;
- §1.7 JSONL/SQLite examples referencing `pi.lane.lastResult`; state the bounded growth tradeoff — one small immutable record per operation, retained forever, carried through JSONL snapshot compaction;
- §2.9 precise rewrite: decide and document the policy for records whose `tipId` the rewrite removes (retain-dangling or delete);
- §3.1–§3.2 state shapes: 13 neutral leaves, `OperationScope`, `SummaryTask`/`ResultBoundary` (kind derived from boundary), `Control` without drained fields, `CheckpointData` without `skipInboxOnce`/`thresholdCheckedTriggerEntryId`;
- §3.5 graph (one summary quadruple, boundary arms);
- §3.6 acceptance: per-tag mode-respecting selection, global admission-order placement, prompt entries last, idle followUp eligibility, and empty-prompt continuation acceptance;
- §3.9/§3.10 summary machinery expressed once over the boundary datum;
- §3.11 complete rewrite: one lane inbox, the selection table, one-decision-one-commit boundaries, silent deferral of late steer (an unconsumed steer becomes future-run input rather than an error), and unbounded write pendency during structural operations with the `waitForIdle`-then-append escape hatch;
- §3.12 checkpoint procedure: the one-commit boundary decision replaces the stepwise algorithm; failures terminalize while queued lane input remains for a later ordinary run;
- §3.13 terminal transactions: result records, universal suffix, observation contract (`drive(id)` total, `getResult`);
- §4.6 abort: drain-and-return, no drained control, and the client-visible consequence — drained steer/followUp payloads exist only in the returned result, so a crash or lost response in that window loses their content permanently;
- §5.1 lane surface and all result types; §5.5 `queue_update` and end-event payloads (`compaction_end` as a segment event with `aborted`);
- Part 9: invariants 12–16, 21, 26 (result-record lifetime, observation via records, continuation equivalence), the new terminal-control invariant (a non-aborted terminal status implies running terminal control), and the race catalog rows touching drained items, `cancelQueued`, and continuation;
- Appendix A glossary (Inbox, Result record, Continuation run, Boundary pass; remove Drained);
- §5.5 event taxonomy: state explicitly which events are operation-terminal (`run_end`, `navigation_end`, compaction-kind `compaction_end`) versus segment brackets (in-run `compaction_start`/`compaction_end`) versus non-terminal lifecycle (`run_suspend` leaves the operation durably open), and name `reduceLaneSnapshot` as the normative fold;
- §2.5 branch-scan sharp edge: `stopAtType` applies after ordering, so `oldestFirst` + `stopAtType: "compaction"` returns the oldest segment; document the canonical context read as `newestFirst` + reverse;
- serving-layer boundary statement: branch-relative reads/appends remain on `AgentLane`, while whole-tree browsing, forks, label inventory, and session listing live beside it in Session/repository services composed by an RPC facade.
Also update `docs/runtime-simplification.md` status and remove every remaining promotion reference in the repository's docs. Grep gate:
```bash
rg -i 'promotion|drainedSteer|drainedFollowUp|laneLastResult|lastResult|skipInboxOnce|thresholdChecked' packages/agent/src packages/agent/docs
```
(matches must be zero in `src`; docs may retain historical notes only inside completed-milestone records.)
## 12. M10 — Provider KV-cache identity review
### Goal
**Implemented and reviewed.** Core Session identity alone is not a valid provider cache lineage: several lanes may issue concurrent requests over divergent transcripts. Ordinary assistant requests therefore derive identity from Session metadata id plus lane name; structural requests remain isolated.
### Review scope
- Inventory every harness and pi-ai path that sets, preserves, derives, or consumes `SimpleStreamOptions.sessionId`, including prompt-cache keys, affinity headers, WebSocket/session-resource caches, deferred requests, and structural summary requests. Ordinary harness generation now forwards the derived lane identity; legacy `src/agent.ts` keeps its independent conversation id; structural summary requests mint fresh ids.
- Define a provider-request cache lineage distinct from the durable Session id and operation id. Concurrent lanes must never share one lineage merely because they belong to the same Session.
- Derive ordinary assistant lineage as `Session metadata id + ":" + lane name`; do not store another durable identifier. It remains stable for the lane's lifetime and differs across lanes in one Session.
- Preserve same-lane reuse across ordinary assistant turns and retries. Compaction, navigation, branch replacement, and model changes may miss an old prefix, but cannot incorrectly reuse it; do not add rotation machinery.
- Keep structural summary requests isolated: `cacheRetention: "none"` and fresh request identity per nested request remain the baseline unless the review proves a safe, useful alternative.
- Distinguish cache identity from observability correlation. Session, lane, operation, task, and request ids remain available to telemetry without being reused blindly as provider affinity/cache keys.
- Review provider-specific semantics rather than assuming `sessionId` means only KV caching; some adapters also use it for headers, WebSocket reuse, fallback state, or resource cleanup.
### Focused validation
- Two active lanes in one Session issue concurrent, divergent prompts and receive distinct provider cache/affinity identities.
- One lane keeps a stable lineage across append-only assistant/tool turns where reuse is valid.
- Context-prefix discontinuities safely miss old cached prefixes without lineage rotation.
- Assistant retries retain the lane identity; deferred handle polling needs no cache identity.
- Structural split-turn requests remain isolated from transcript assistant caching.
- Faux-provider tests assert stable same-lane and distinct cross-lane request identities.
### Exit condition
Ordinary assistant requests use the derived lane identity, structural requests keep fresh identities with `cacheRetention: "none"`, and deferred handle polling sends no cache identity. The policy is documented in `docs/harness.md` and independently reviewed before WP05 completion.
## 13. Mobile assistant-output handoff
**Tracked follow-up; not part of the M8/M10 public-drive gate.** A trivial mini coding-agent Session produced an approximately 300 KB JSONL file because live assistant streaming persists many `pi.pending.assistant_frame` list appends. Logical frame cleanup does not reclaim bytes already appended to the JSONL history, so short conversations can have disproportionate durable storage and replay cost.
The authoritative follow-up is the [mobile assistant-output handoff](../mobile-handoff/01-harness/05-assistant-output/message-update.md) and its numbered prerequisites in the [mobile handoff README](../mobile-handoff/README.md). It addresses the complete path rather than only batching frames: Chord op tracking, scoped pending-output durability, tool/assistant output reduction, and `message_update` replication amplification. Preserve the existing crash contract: an admitted assistant effect remains reconstructible, progress observation remains useful, and settlement retires operation-owned pending state.
Exit checks:
- representative short streamed responses do not create hundreds of kilobytes of durable frame history;
- crash/reopen at every assistant and deferred effect boundary reconstructs the same pending message;
- frame/checkpoint writers remain fenced to the owning durable effect;
- backend conformance covers both reconstruction and bounded write amplification.
## 14. Module boundaries
```text
session/** durable storage; imports no runtime module
execution/** neutral provider/tool/gate mechanics
runtime/types.ts LaneState, Drive, command decisions
runtime/progress.ts frame/tool progress channels
runtime/drive/*.ts direct procedures
runtime/drive.ts total flat switch, created only in M7
runtime/lane.ts Lane actor and public surfaces
runtime/harness.ts Harness lifecycle and Lane composition
```
Procedure modules import concrete `Lane<TContext>` type-only. `TContext` remains `object | undefined` invariant. No `any`, `Lane<any>`, `as unknown as`, `@ts-expect-error`, inline imports, parameter properties, enums, or other non-erasable TypeScript syntax.
## 15. Exclusions
Do not introduce:
- generic scheduler, graph, action interpreter, or effect-plan DSL — `ResultBoundary` stays a closed three-arm data union switched in one visible place;
- per-family outcome unions, `OptionalFinalAssistant`, or embedded result entries in stored records;
- result-record listing, filtering, pagination, status queries, or retention machinery;
- operation-owned queues, drained-control fields, or terminal queue cleanup;
- automatic continuation anywhere below the convenience layer (never inside `drive`);
- a second mutation line or transaction framework;
- expected-`at` runtime checks for the sole top-level writer;
- process-local Drive replacement or caller-owned lifetime;
- external finalization;
- storage rereads of authoritative control state;
- read caches, read budgets, or generic `getValues` batching;
- compatibility aliases for any pre-redesign durable shape;
- a `phase`/segment discriminator field on structural events — the reducer derives segment-vs-terminal from the open operation's kind;
- whole-tree, fork, label-inventory, or repository methods on `AgentLane`; branch-relative reads/appends remain part of the lane facade, while broader administration lives beside it.
Procedure-specific writes, effect admission, settlement classification, and event construction remain visible at their call sites.
## 16. Validation and reviews
After every code stage:
```bash
npm run check
```
Run each modified focused test file from the package root. Do not invoke the full Vitest suite directly. Run `./test.sh` only for final package validation or when explicitly requested.
Review checkpoints are mandatory at the end of R3, R1b, M7, M10, and final completion. Delegated reviews use provider `anthropic` and model `claude-fable-5`.
Final greps:
```bash
rg 'lost_ownership|LostOwnership|DriveAbandoned|commandDriveOwned|installerSignal|finalizedOutcome|OperationEnded' \
packages/agent/src/harness packages/agent/test/harness
rg 'drainedSteer|drainedFollowUp|laneLastResult|RunOutcome|CompactionOutcome|NavigationOutcome|TerminalOperationOutcome|skipInboxOnce|thresholdCheckedTriggerEntryId' \
packages/agent/src/harness
rg 'SliceNotImplemented' packages/agent/src/harness/runtime
rg ': any\b|<any>|as unknown as|@ts-expect-error' packages/agent/src/harness/runtime
```
Final exit conditions:
- every flat leaf is driveable and reconcilable;
- public primitive/convenience behavior is equivalent, including continuation;
- every focused test and backend conformance path passes;
- public drive exposes no partial graph;
- `watchSession` is the only deferred public method;
- `drive(id)` answers every settled operation id in the session;
- harness.md is self-consistent with the implementation (M9);
- provider cache/affinity identity is lane-safe across concurrent and context-reset histories (M10);
- independent final review reports no blocker.

734
third_party/pi/packages/agent/docs/work-packages/06-session-branch-lane-separation.md vendored
View File

@@ -1,734 +0,0 @@
# WP06 — Session, Branch, Lane separation
**Status: implemented before WP05 M4.** WP05 M3 remains complete. Retry/deferred work resumes on the composition, mutation, and ownership boundaries landed here.
This package replaces the mixed `SessionTree`/implicit-main inheritance design with four explicit concepts:
```text
Session global durable data + one mutation line
Branch one path through the entry tree, with a movable tip
AgentLane Branch data surface + agent operations/configuration
AgentHarness manager of AgentLanes; never a lane itself
```
---
## 0. Mandatory reading
Read completely before editing:
1. `packages/agent/docs/harness.md`.
2. `packages/agent/docs/work-packages/05-direct-durable-drive.md`.
3. `packages/agent/src/harness/session/types.ts`.
4. `packages/agent/src/harness/session/session.ts`.
5. `packages/agent/src/harness/session/memory.ts`.
6. `packages/agent/src/harness/session/jsonl/repo.ts` and `jsonl/storage.ts`.
7. `packages/session-backends/sqlite-node/src/sqlite/session.ts`, `storage.ts`, and repo implementation.
8. `packages/agent/src/harness/runtime/lane.ts`, `harness.ts`, `restore.ts`, and `types.ts`.
9. `packages/agent/src/harness/agent-harness.ts`.
10. `packages/agent/src/harness/session/fork.ts` and repository conformance tests.
11. Every test named in §8 before modifying it.
Do not inspect deleted runtime implementations or Git history. Current source, `harness.md`, WP05, and this package are the only sources of truth.
---
## 1. Problem
### 1.1 `SessionTree` mixes unrelated ownership
`SessionTree` currently contains:
```ts
interface SessionTree {
// Lane/path data.
getLeafId(...): Promise<string | null>;
findEntriesOnBranch(...): Promise<Entry[]>;
findEntryOnBranch(...): Promise<Entry | undefined>;
appendMessage(...): Promise<string>;
appendCustomEntry(...): Promise<string>;
// Session-global data that is not lane- or branch-owned.
getEntry(...): Promise<Entry | undefined>;
getStats(...): Promise<SessionStats>;
findEntries(...): Promise<Entry[]>;
findEntry(...): Promise<Entry | undefined>;
getValue(...): Promise<StoredValue<unknown> | undefined>;
setValue(...): Promise<void>;
readList(...): Promise<ListElement<unknown>[]>;
appendList(...): Promise<void>;
getName(...): Promise<string | undefined>;
setName(...): Promise<void>;
getLabel(...): Promise<string | undefined>;
setLabel(...): Promise<void>;
}
```
A selected tree view changes only which mutation queue a global write enters. It does not change the durable address. Two views can therefore read the same global value under different lane lines and commit a lost update.
### 1.2 `Session` silently means `main`
`Session extends SessionTree`. Its inherited branch methods and high-level writes silently delegate to the `main` lane. `session.setValue(...)` is actually `setValueForLane("main", ...)`. A fresh repository session creates a partial implicit main lane before any harness exists.
### 1.3 `AgentHarness` silently means `main`
`AgentHarness extends AgentLane`, and the runtime `Harness extends Lane`. The manager object is inserted into its own lane map as `main`. Calls such as `harness.prompt(...)`, `harness.watch(...)`, or `harness.getModel(...)` silently target main.
### 1.4 Per-lane mutation queues solve the wrong problem
Storage already serializes atomic commits and assigns one session-wide `seq` per write. Per-lane mutation queues allow useful preparation overlap, but this runtime does not need that complexity now: all harness mutation callbacks perform bounded storage reads, prepare one write set, commit at most once, publish process-local state, and return. Providers, tools, hooks, timers, and asynchronous event delivery are outside mutation callbacks.
The approved first implementation uses one Session mutation line. Keyed lines may be added later if profiling proves the global line is a bottleneck; that future change requires a mutable-ownership audit but no public API redesign.
---
## 2. Approved terminology and ownership
### Session
Owns:
- session metadata;
- global entry and usage queries;
- application values and lists;
- session name and entry labels;
- Branch discovery/creation;
- one process-local mutation line for every supported mutation;
- one open storage/backend lifecycle.
A Session does not implement Branch and has no implicit branch.
### Branch
Data-only capability describing one named path through the immutable entry tree.
Owns only:
- its current tip;
- branch-relative entry queries;
- direct extension of its tip with message or custom entries.
A Branch has no model configuration, queues, operation state, drive, hooks, or agent policy.
### AgentLane
One Branch plus agent configuration and operations. It exposes Branch methods directly rather than exposing a nested `Branch`, tree, store, view, or access object.
When idle, `AgentLane.appendMessage` / `appendCustomEntry` extend the tip directly. During an active run, they retain the existing deferred-write semantics: reserve the entry id, persist `pendingEntry(id)`, and enqueue the id in the operation inbox for checkpoint placement. A raw Branch append is always a direct data append; raw Branch mutation while a Harness owns the corresponding lane is a trusted-programming defect.
### AgentHarness
Owns global registries/configuration, hooks, events, lifecycle, and a map of AgentLanes. It is not an AgentLane and exposes no implicit-main operation methods.
---
## 3. Target public types
### 3.1 Session reader and mutation
```ts
export interface SessionReader {
getEntries(ids: string[], context: Context): Promise<Map<string, Entry>>;
getValue<T>(
address: Value<T>,
context: Context,
): Promise<StoredValue<T> | undefined>;
scanValues<T>(prefix: Value<T>, context: Context): Promise<StoredValue<T>[]>;
readList<T>(
address: ValueList<T>,
options: ListReadOptions | undefined,
context: Context,
): Promise<ListElement<T>[]>;
scanBranch(query: StorageBranchScan, context: Context): Promise<Entry[]>;
}
export interface SessionMutation extends SessionReader {
/** Exactly zero or one attempt. A second call rejects, including after failure. */
commit(writes: Write[], context: Context): Promise<CommitResult>;
/** Wait for any admitted commit, invalidate the capability, and release the Session line. */
end(context: Context): Promise<void>;
}
export type SessionMutator = Omit<SessionMutation, "end">;
export type SessionMutationCallback<TResult> = (
mutator: SessionMutator,
context: Context,
) => TResult | Promise<TResult>;
```
`beginMutation()`/`SessionMutation.end()` remain the transportable scope used by the current remote Session protocol. They are keyless and carry no lane field. Normal local/harness code uses `mutate()`; no caller selects a mutation key.
### 3.2 Branch
```ts
export interface Branch {
readonly name: string;
getTipId(context: Context): Promise<string | null>;
findEntries(
query: BranchScan | undefined,
context: Context,
): Promise<Entry[]>;
findEntry(
query: BranchScan | undefined,
context: Context,
): Promise<Entry | undefined>;
appendMessage(message: AgentMessage, context: Context): Promise<string>;
appendCustomEntry(
customType: string,
data: JsonValue | undefined,
context: Context,
): Promise<string>;
}
```
Because the receiver is already a Branch, the public names are `findEntries` and `findEntry`, not `findEntriesOnBranch` and `findEntryOnBranch`.
### 3.3 Session
```ts
export interface Session<
TMetadata extends SessionMetadata = SessionMetadata,
> extends SessionReader {
readonly metadata: TMetadata;
readonly idGenerator: IdGenerator;
// Direct reads. No mutation-line acquisition.
getEntry(id: string, context: Context): Promise<Entry | undefined>;
getStats(context: Context): Promise<SessionStats>;
findEntries(
query: EntryQuery | undefined,
context: Context,
): Promise<Entry[]>;
findEntry(
query: EntryQuery | undefined,
context: Context,
): Promise<Entry | undefined>;
getName(context: Context): Promise<string | undefined>;
getLabel(targetId: string, context: Context): Promise<string | undefined>;
// Existing Branch acquisition performs durable I/O and therefore receives Context.
branch(name: string, context: Context): Promise<Branch | undefined>;
createBranch(
name: string,
at: string | null,
context: Context,
): Promise<Branch>;
// Transportable explicit scope; RemoteSession maps begin/read/commit/end over RPC.
beginMutation(context: Context): Promise<SessionMutation>;
// Trusted sharp edge. The callback holds the sole Session mutation line.
mutate<TResult>(
mutation: SessionMutationCallback<TResult>,
context: Context,
): Promise<TResult>;
// One-write conveniences implemented through mutate().
setValue<T>(
address: Value<T>,
next: NoInfer<T>,
context: Context,
): Promise<void>;
deleteValue<T>(address: Value<T>, context: Context): Promise<void>;
appendList<T>(
address: ValueList<T>,
element: NoInfer<T>,
context: Context,
): Promise<void>;
deleteList<T>(address: ValueList<T>, context: Context): Promise<void>;
setName(name: string | undefined, context: Context): Promise<void>;
setLabel(
targetId: string,
label: string | undefined,
context: Context,
): Promise<void>;
close(context: Context): Promise<void>;
}
```
`mutate()` remains public. Plugins are trusted not to retain the mutator, invoke nested public writers, perform effects, or hold the line across unbounded work. Misuse may block every mutation in that Session and is a plugin defect. `beginMutation()` exists for transport/lifecycle integration rather than ordinary plugin work; every direct caller must call `end()` in `finally`.
### 3.4 AgentLane
`AgentLane` keeps its operation/configuration/observation methods and directly adds the Branch surface:
```ts
export interface AgentLane {
readonly name: string;
getTipId(context: Context): Promise<string | null>;
findEntries(
query: BranchScan | undefined,
context: Context,
): Promise<Entry[]>;
findEntry(
query: BranchScan | undefined,
context: Context,
): Promise<Entry | undefined>;
appendMessage(message: AgentMessage, context: Context): Promise<string>;
appendCustomEntry(
customType: string,
data: JsonValue | undefined,
context: Context,
): Promise<string>;
getLastResult(context: Context): Promise<LaneLastResult | undefined>;
accept(
request: OperationRequest,
context: Context,
): Promise<OperationAdmissionResult>;
drive(options: DriveOptions, context: Context): Promise<DriveResult>;
requestAbort(
operationId: string,
context: Context,
): Promise<AbortRequestResult>;
inspectExecution(context: Context): Promise<LaneExecutionInfo>;
// Existing convenience, queue, configuration, idle, and watch methods remain.
}
```
Delete `AgentLane.sessionTree`.
### 3.5 AgentHarness
```ts
export interface AcquireLaneOptions {
/** Used only when the AgentLane does not exist. Defaults to null. */
createAt?: string | null;
}
export interface AgentHarness<
TContext extends object | undefined = object | undefined,
> {
lane(name: string, context: Context): Promise<AgentLane>;
lane(
name: string,
options: AcquireLaneOptions,
context: Context,
): Promise<AgentLane>;
lanes(context: Context): Promise<LaneInfo[]>;
// Session-global metadata wrappers preserve existing value_update events.
getName(context: Context): Promise<string | undefined>;
setName(name: string | undefined, context: Context): Promise<void>;
getLabel(targetId: string, context: Context): Promise<string | undefined>;
setLabel(
targetId: string,
label: string | undefined,
context: Context,
): Promise<void>;
// Existing global tools/resources/options/settings/hooks/events/watchSession/close surface.
}
```
`AgentHarness` does not extend `AgentLane`. Delete `createLane`; `lane()` is atomic get-or-create. Existing lanes ignore `createAt`. A missing lane uses `createAt ?? null`. Concurrent acquisitions return the same published AgentLane. Invalid names and unknown non-null targets reject with the existing tagged errors; close/fault reject with their existing lifecycle errors.
A fresh Session and fresh Harness contain no implicit main Branch or AgentLane. `await harness.lane("main", context)` creates main completely. `lanes()` may return `[]`.
---
## 4. Mutation and read semantics
### 4.1 One Session mutation line
Replace `LaneMutationLine` with a single `MutationLine`:
```ts
export class MutationLine {
private tail: Promise<void> = Promise.resolve();
private sealedError: Error | undefined;
run<TResult>(operation: () => TResult | Promise<TResult>): Promise<TResult>;
seal(error: Error): Promise<void>;
}
```
`StorageBackedSession.beginMutation()` acquires that line and returns one explicit keyless capability; only `end()` releases it. `mutate()` is the callback convenience built from begin/end and always ends in `finally`. The callback may read, prepare, commit once, publish process-local state, synchronously bind event recipients, and return. `close()` seals admission and waits for every acquired scope to end before closing Storage.
High-level Session writes, Branch creation/appends, every `Lane.command`, progress writes, restore snapshots requiring coherence, and Harness lane acquisition all call the same `Session.mutate()`.
### 4.2 Reads bypass the line
Ordinary Session and Branch reads call Storage directly. They observe the latest fully applied atomic storage commit at the time each read executes:
- a queued/planning/unapplied mutation is invisible;
- no partial commit is visible;
- once Storage commit resolves, direct reads may observe it even while the mutation callback is still publishing process-local state;
- multiple reads outside one `mutate()` are not a snapshot;
- coherent read-decide-write/CAS uses `Session.mutate()`.
### 4.3 Effects stay outside
A mutation callback must not perform or await:
- providers or deferred fetch/cancel;
- tools;
- hooks;
- timers;
- asynchronous event delivery;
- idle callbacks or Drive completion;
- nested Session/Branch/AgentLane mutators.
The callback may synchronously call `emitBatch` after publication to bind recipients and retain its delivery promise. The public operation awaits delivery after `mutate()` returns.
### 4.4 Storage remains independently atomic
Storage retains its one-session commit serializer and assigns one global `seq` per write. The Session mutation line protects read-decide-commit-publication procedures; the Storage queue protects atomic transaction application, sequence assignment, fork snapshots, and backend callers. Do not merge the two abstractions.
---
## 5. Branch and lane durable shape
### 5.1 No implicit main
Repository `create()` writes only session metadata/header/catalog state. It writes no branch tip, lane configuration, or lane state. Remove main seeding from Memory and JSONL creation and from SQLite initialization.
Legacy coding-agent v3 normalization may still produce a main Branch because the imported transcript has one selected path.
### 5.2 Branch completeness
A Branch exists exactly when its required tip value exists. `createBranch(name, at, context)` validates name, absence, and non-null target, then writes the tip in one mutation. It writes no model configuration or operation state.
Use Branch terminology in source. This package renames the typed constructor and public concepts to `branchTip`/`tipId`. The persisted namespace and durable field spelling decision must be consistent across all backends and docs:
- use `pi.branch.tip` and rename format-4 fields from leaf to tip rather than retain misleading new-code aliases;
- format 4 and the new harness are WIP, so replace their schema and field names in place: no storage-version bump, migration, compatibility decoder, or old-format rejection path;
- legacy coding-agent v3 import remains supported: it maps its selected main leaf to a main Branch tip and walks that selected physical ancestry to reconstruct the nearest `model_change`, `thinking_level_change`, and `active_tools_change` independently; unsupported nearest values do not fall back to older history;
- when the importer can reconstruct a total configuration, it writes ordinary `laneConfig("main")` plus fresh idle `laneState("main")` before returning the Session; missing active-tools history normalizes to `[]` because v3 did not persist the initial tool inventory;
- if required model or thinking configuration is missing or unsupported, the importer leaves a data-only main Branch rather than persisting partial compatibility state;
- update the legacy active-tools record type to read its encoded `activeToolNames` array;
- update protocol schemas and experimental adapters for public `tipId` fields in the same package.
Inventory uses `branchTipInventoryPrefix()`. Legacy import emits only ordinary Branch/Lane values; there is no temporary compatibility address or attachment-time migration.
### 5.3 AgentLane completeness
An AgentLane adds total `laneConfig`, `laneState`, optional `laneLastResult`, and optional current operation values to an existing Branch.
`harness.lane(name, options?, context)` executes one Session mutation and handles exactly these cases:
| Durable state | Result |
| ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Branch absent; lane values absent | validate `createAt`, commit Branch tip + the immutable `AgentHarnessOptions` seed config + idle lane state, publish one AgentLane and `lane_created { at: createAt ?? null }` |
| Branch present; lane config/state absent and no last result | commit the immutable seed config + idle lane state at the existing tip, publish one AgentLane and `lane_created { at: existingTip }` |
| Branch and complete lane values present | return the restored/published AgentLane; no commit/event |
| Any partial or contradictory combination | fault as storage corruption |
The committing callback publishes the new Branch/AgentLane into process-local maps and synchronously calls `emitBatch(lane_created, context)` before returning from `Session.mutate()`. Event delivery is awaited after line release.
`AgentHarness.create()` restores every complete durable AgentLane and open operation, but creates nothing and requires no main. A data-only Branch remains a Branch until `harness.lane(name, ...)` attaches agent state.
### 5.4 Append behavior
`Branch.appendMessage` / `appendCustomEntry` always commit an immutable entry parented to the current tip and move the tip in the same transaction.
`AgentLane.appendMessage` / `appendCustomEntry` retain harness semantics:
- idle: append and move tip immediately;
- active run: stage `pendingEntry(id)` and enqueue `inbox.writes`;
- active structural operation: retain the existing wait/re-evaluate contract when that surface lands;
- pending assistant messages reject before commit.
Both return the entry id reserved before their mutation.
---
## 6. Runtime composition
### 6.1 Harness
Replace inheritance with composition:
```ts
export class Harness<
TContext extends object | undefined,
> implements AgentHarness<TContext> {
readonly session: Session;
readonly models: Models;
readonly hooks: HookRegistry;
readonly events: HarnessEventBus;
readonly lanesByName = new Map<string, Lane<TContext>>();
// global config/lifecycle fields
}
```
The constructor builds every restored Lane as an ordinary object. It never calls `super("main", ...)` and never inserts `this` into `lanesByName`.
Fault and close iterate ordinary Lane objects. Global getters/setters live only on Harness. Coding-agent experimental services and workers must acquire/cache `main` explicitly before invoking lane operations.
### 6.2 Lane
`Lane.command` and `commandDriveOwned` call keyless `session.mutate(plan, context)`. The live `Lane.state` remains the authoritative process projection. Exact Drive fencing remains adjacent to commit admission.
Lane directly implements Branch query/append names. It may hold a package-private Branch implementation for direct data reads, but no nested Branch is exposed publicly.
### 6.3 Restore
Restore no longer enters a named mutation line. `AgentHarness.create()` owns the Session attachment interval and performs one bounded keyless `Session.mutate()` callback to inventory/restore every complete AgentLane before publishing the Harness; it commits only when an intentional attachment normalization is required. Coherent live watch/inspection likewise uses `Session.mutate()` as a no-commit callback.
Restore inventories complete AgentLanes separately from data-only Branches. Missing main is legal.
---
## 7. Repository, backend, and fork requirements
### Memory/JSONL/SQLite
All Session facades:
- remove lane arguments and lane fields from begin/mutate capabilities;
- retain explicit `beginMutation(context)` / `SessionMutation.end(context)` forwarding for local lifecycle and remote transport;
- keep an explicit scope admitted until `end()` and keep callback `mutate()` admitted through its implicit finally/end;
- expose Branch acquisition/creation;
- keep direct reads outside the line;
- keep Storage commit sequencing unchanged.
SQLite and future SQL backends still serialize sequence-allocating commits in Storage. The Session mutation line intentionally serializes complete callbacks only within one open Session owner; separate sessions remain concurrent.
### Forks
Preserve one coherent source Storage snapshot. Update terms from lane leaf to Branch tip.
- branch-scope fork creates destination Branch `main` at the copied path;
- if no explicit source entry is provided, source `main` must exist or fork rejects;
- branch scope copies source main's configuration and writes idle lane state **together iff** source main is a complete configured AgentLane; an unconfigured/data-only source main produces only a data-only destination main Branch;
- tree scope copies every Branch tip; each complete configured source AgentLane copies its configuration plus fresh idle lane state together, while each data-only Branch remains data-only;
- operation values, pending values/lists, last results, and usage rows remain excluded;
- destination sessions do not gain an unrelated implicit main;
- retain the explicit begin/commit/end fork-ordering seam: code that must admit a commit before starting a fork snapshot calls keyless `beginMutation()`, invokes `commit()`, starts the repository snapshot only after commit admission, and calls `end()` in `finally`; Storage queues the source snapshot against commits to choose one coherent boundary.
---
## 8. Implementation phases and file manifest
Public drive remains disabled. Complete this package before resuming WP05 M4.
### Phase A — Session mutation and Branch
**Rename**
- `src/harness/session/lane-mutations.ts` → `mutation-line.ts`.
**Modify**
- `src/harness/session/types.ts` — `Branch`, keyless `Session.mutate` and keyless begin/end transport scope, no `SessionTree`, Session-global methods.
- `src/harness/session/session.ts` — one line, Branch implementation, no implicit-main delegates, direct Branch append; rename lane-creation validation errors to Branch terminology.
- Memory/JSONL/SQLite format-4 schema and codecs — replace WIP leaf/lane spellings in place; do not add a version gate or migration.
- `src/harness/session/memory.ts`.
- `src/harness/session/jsonl/repo.ts`, `jsonl/legacy-v3.ts`, and related open/create facade files.
- `packages/session-backends/sqlite-node/src/sqlite/session.ts` and repo creation.
- `src/harness/session/fork.ts`.
- `src/harness/session/index.ts` and package exports.
- storage/repository benchmarks and conformance call sites.
**Rename tests**
- `test/harness/session-tree.test.ts` → `branch.test.ts`.
- `test/harness/session-create-lane.test.ts` → `session-create-branch.test.ts`.
**Modify tests**
- `test/harness/storage-backed-session.test.ts`.
- `test/harness/memory-session-repo.test.ts`.
- `test/harness/jsonl-session-repo.test.ts`.
- `test/harness/memory-conformance.test.ts`.
- `src/harness/session/testing/conformance/session-repo.ts`.
- SQLite repo/storage tests.
- compaction/branch-summarization type fixtures.
### Phase B — Harness/Lane composition
**Modify**
- `src/harness/agent-harness.ts`.
- `src/harness/runtime/harness.ts`.
- `src/harness/runtime/lane.ts`.
- `src/harness/runtime/restore.ts`.
- `src/harness/runtime/types.ts` where `leafId` becomes `tipId`.
- `src/harness/session/values.ts` and durable state types for Branch tip naming.
- all runtime drive modules already present (`checkpoint`, `generation`, `recovery`, `terminal`, `progress`) only where names/signatures change.
- `src/harness/compaction/branch-summarization.ts`.
- `packages/protocol/src/harness.ts`.
- `packages/coding-agent/src/experimental/services/agent-controller-provider.ts`.
- `packages/coding-agent/src/experimental/services/models-provider.ts`.
- `packages/coding-agent/src/experimental/session-worker.ts`.
- experimental harness wire/session worker tests.
**Modify focused tests**
- every `test/harness/runtime/*.test.ts` helper/call site using keyed mutate, implicit Harness-as-main, `sessionTree`, `leafId`, or `createLane`;
- `test/harness/types.test.ts`;
- `test/harness/branch-summarization.test.ts`;
- protocol and coding-agent experimental tests.
Phases A and B are one atomic landing. Removing `SessionTree`, keyed mutate, and Harness inheritance cannot compile as separately committed compatibility phases, and this package intentionally adds no temporary aliases.
### Phase C — Normative documentation
Update `packages/agent/docs/harness.md` completely and consistently:
- orientation/system model and worked examples;
- bound Branch tip addresses;
- Branch, Session metadata, queries, forks, and repository boundary;
- operation metadata/result/snapshot `tipId` terminology;
- one Session mutation line in Parts 3–5;
- attachment with no required main;
- public Branch, AgentLane, AgentHarness, and Session surfaces;
- event/watcher ordering under the Session line;
- remove the old second harness-settings-line lock-order narrative: harness-global settings and every durable lane mutation now serialize through the sole Session line when a coherent snapshot is required; pure synchronous registry reads remain direct;
- work-package table, invariants, races, backend conformance, and glossary.
Update WP05 before M4:
- replace `SessionTree`/lane-line/inheritance assumptions;
- replace `leafId` source examples where the new type names require it;
- keep all M0–M3 historical behavior and M4–M8 durable requirements;
- state that WP06 is the foundation between M3 and M4.
Update every current supporting document whose public names or ordering statements change:
- `docs/assistant-durability.md` and `docs/tool-durability.md` — Session-line FIFO and `branchTip` terminology;
- `docs/values.md` — Session-global value/list surface, Branch tip addresses, and no `SessionTree`;
- `docs/telemetry.md` — receiver inventory and Session mutation spans;
- `docs/plugins.md` — remove `sessionTree`; the plugin's AgentLane directly supplies Branch methods, while a scoped Session-data facet supplies global value/list/name/label/query methods and excludes raw `mutate`, `idGenerator`, close, and backend authority;
- `docs/extensions/pi-extensions-v2.md` and `docs/extensions/pi-server-artifact/index.md` where examples/types use the changed surfaces;
- completed WP00–WP04 only where a forward-looking/current-state statement would otherwise claim the removed API still exists.
Retain the current remote Session mutation contract and update it from a named lane line to the sole Session line: worker `RemoteSession.mutate()` performs keyless begin RPC → local callback with remote reads/one remote commit → local post-commit publication → end RPC. The server holds the Session line through commit and publication until end acknowledgment. Disconnect/timeout terminates the scope under the existing hosting policy. Update the current remote protocol/vertical-slice documentation and every implementation/test present on dev; do not delete or defer this behavior.
Do not rewrite released changelog sections. Add no changelog entry on a non-main/non-PR development branch.
---
## 9. Required tests
### Session mutation line
- concurrent `mutate()` callbacks serialize globally, including callbacks invoked from different AgentLanes;
- a second callback does not enter until the first callback returns after commit/publication;
- direct reads do not wait for a callback that has not committed;
- direct reads observe a fully landed commit even while that mutation callback remains open after commit;
- no direct read observes a partial multi-write commit;
- two read-modify-write counter mutations produce `1`, then `2`;
- separate direct `getValue` + `setValue` calls remain deliberately non-atomic;
- zero-commit callback is legal;
- keyless `beginMutation()` excludes every other mutation until `end()`, commit does not release it, end-without-commit is legal, repeated end is idempotent, and close waits for end;
- RemoteSession begin/read/commit/publication/end preserves that same scope;
- second commit attempt rejects, including after a failed first attempt;
- nested public write from a mutation callback is documented as invalid and deterministically blocks/rejects according to the chosen guard;
- close-first rejects mutation; mutation-first completes and close waits; a trusted callback that never returns can block close;
- Storage commit `seq` and stats behavior is unchanged.
### Session and Branch
- fresh Session has zero Branches and no main tip value;
- no storage-version bump, migration, compatibility decoder, or rejection path is added for the replaced WIP format-4 schema;
- legacy coding-agent v3 import reconstructs ordinary total main-lane configuration plus idle state before returning when valid model/thinking history exists; otherwise it returns a data-only main Branch;
- legacy config tests cover complete and incomplete histories, invalid legacy fields, and branched histories where only changes on the selected main path apply;
- `branch(name, context)` returns undefined for absence and receives the exact Context on reads;
- `createBranch` validates name, target, and duplicate atomically;
- two concurrent creates have one winner;
- Branch queries default to its tip and preserve scan/filter/cursor behavior;
- direct Branch message/custom append extends its tip atomically;
- custom entry with absent data remains valid;
- pending assistant append rejects;
- Session global values/lists/name/labels/global queries no longer depend on a branch;
- all pre-close Branch objects reject after close;
- branch-scope fork copies config + idle state together iff source main is configured; tree scope does the same independently per configured lane; data-only Branches remain data-only;
- explicit keyless begin/commit/end preserves commit-before-fork-snapshot ordering and captures one Storage-serialized snapshot boundary.
### AgentHarness and AgentLane
- Harness has no AgentLane methods at type or runtime;
- Session has no Branch methods at type or runtime;
- AgentLane has Branch methods directly and no `sessionTree`/nested Branch property;
- fresh Harness `lanes()` is empty;
- `lane("main")` creates complete main atomically;
- missing named lane defaults to null tip; `createAt` anchors creation;
- existing data-only Branch becomes one complete AgentLane without moving its tip;
- existing complete AgentLane acquisition commits nothing and emits no creation event;
- concurrent acquisitions return the same object and emit exactly one `lane_created`;
- creation commit publishes the Lane and binds recipients before releasing `Session.mutate`;
- invalid name/unknown anchor commit nothing;
- restored complete lanes/open operations are inventoried without creating main;
- partial Branch/config/lane-state combinations fault;
- Harness global metadata wrappers retain commit → publication → `value_update` delivery;
- AgentLane idle append moves the tip; active-run append stages one deferred write and preserves authoritative state;
- close/fault seal every ordinary Lane without relying on Harness inheritance.
### Regression
- M2 exact-Drive ABA fence still checks immediately before commit admission;
- M3 generation intent/frame/settlement writes are byte-identical apart from renamed durable/public fields;
- frame FIFO remains correct under the one Session line;
- watch has only snapshot-first or publication-first outcomes;
- usage totals remain commit-boundary exact across lanes;
- Context remains trailing and source-identical throughout;
- Memory, JSONL, and SQLite repository/storage conformance pass.
---
## 10. Exclusions
Do not add:
- keyed mutation lines, arbitrary lock names, resource locks, multi-lock ordering, versions, or optimistic retries;
- a scheduler, transaction framework, action interpreter, or generic post-commit task system;
- a nested Branch/tree/store/access property on AgentLane;
- compatibility aliases for `SessionTree`, `view`, implicit Session main methods, Harness lane methods, or `createLane`;
- keyed/named begin/end RPC scopes or removal of the current keyless RemoteSession mutation transport;
- effects inside `Session.mutate()`;
- a second Storage commit/sequence mechanism;
- automatic work start from `AgentHarness.create()` or `harness.lane()`;
- public drive before WP05 M8.
Do not modify provider/tool behavior, durable execution phases, retry/deferred policy, or assistant-frame semantics beyond signature/name propagation required by this package.
---
## 11. Validation
Run every modified focused test, then:
```bash
git diff --check
npm run check
./test.sh
rg -n "SessionTree|sessionTree|\.view\(|LaneMutationLine|extends AgentLane|extends Lane" \
packages/agent/src packages/agent/test packages/agent/docs \
packages/session-backends packages/protocol packages/coding-agent/src/experimental \
packages/coding-agent/test/experimental*
rg -n "beginMutation\([^)]*,|mutate\([^)]*,[^)]*," \
packages/agent/src packages/agent/test packages/session-backends packages/coding-agent/src/experimental
rg -n "session\.mutate\([^)]*\"|\.mutate\(\s*[a-zA-Z_$][a-zA-Z0-9_$]*\s*," \
packages/agent/src packages/agent/test packages/session-backends
```
Expected first grep matches only this package's problem statement, explicitly historical work-package API descriptions, negative type assertions, and unrelated coding-agent tree-widget names such as `SessionTreeNode`; no current harness `SessionTree` concept remains. The keyed-mutate grep must be empty after manually checking false positives.
Review the complete implementation and normative doc update with Fable before committing. Do not commit without explicit user approval.
---
## 12. Stop condition
WP06 is complete when:
- Session has one keyless mutation line; callback `mutate()` and explicit begin/commit/end remote transport share it, and neither accepts a lane key;
- direct reads bypass that line and expose only fully applied Storage commits;
- `SessionTree` and implicit-main Session behavior are gone;
- Branch is the data-only path/tip abstraction;
- AgentLane exposes Branch methods directly and retains operation-aware append semantics;
- AgentHarness is composition-only, has no AgentLane methods, and atomically gets/creates lanes;
- fresh Session/Harness require no main;
- all process-local publication and event-binding boundaries remain correct;
- all three backends and focused race tests pass;
- `harness.md` and WP05 describe the new model consistently;
- final Fable review reports no findings;
- WP05 M4 may resume.

372
third_party/pi/packages/agent/docs/work-packages/07-sqlite-host-ownership-live-forks.md vendored
View File

@@ -1,372 +0,0 @@
# WP07 — SQLite host ownership and live-source forks
**Status: implemented.**
The delivered backend has no writer lease or replacement ownership primitive. It provides no-create read-write/read-only opens, queued same-repository snapshots, independent read-only WAL snapshots for live external sources, canonical physical identity, path-safe IDs, repository-local deletion reservation, and all-settled close. The tests cover both per-file and shared-container layouts, including a writer commit completed after a read snapshot boundary but before that reader closes.
This package aligns `packages/session-backends/sqlite-node` with the product ownership model: the server owns Session records and worker lifecycle, and exactly one host-assigned process owns writable Session authority at a time. Normally that process is the Session worker. The server may temporarily own a newly created or forked destination, but it closes that Session before handing its metadata to a worker.
Storage does not implement writer ownership. Remove the SQLite writer lease; do not repair or replace it.
A server-side fork is intentionally different from a second writer: it may open a live worker-owned source concurrently for one coherent read-only snapshot while the worker keeps committing. Shared SQLite containers remain supported.
## 0. Mandatory reading
Read completely before editing:
1. `packages/agent/docs/plugins.md` ownership, replacement, and removal sections.
2. `packages/server/README.md` and relevant Session routing/removal source and tests.
3. `packages/agent/docs/harness.md` §§0.6, 1.4–1.7, 2.7–2.8, 4.3, and Part 9.
4. `packages/agent/docs/post-wp05-roadmap.md`.
5. completed WP06 §7 and repository/fork conformance.
6. every source file under `packages/session-backends/sqlite-node/src`.
7. every test and benchmark under `packages/session-backends/sqlite-node`.
8. `packages/session-backends/sqlite-node/README.md` and `CHANGELOG.md`.
Do not use stale files under `dist/` as implementation input. Completed WP01/WP06 documents are historical; do not rewrite them to hide the earlier lease implementation.
## 1. Fixed architecture
### 1.1 Writable authority belongs to the host
Exactly one host-assigned process owns a writable Session. The Session worker is the normal owner. Worker replacement closes the old owner before the new worker opens the Session. Server management serializes creation, forking, removal, and attachment lifecycle around that ownership transfer.
Memory, JSONL, and SQLite do not detect a second process opening the same Session for writes. Bypassing the server/worker lifecycle is a trusted-host defect, not a storage race to repair. A repository still rejects duplicate writable handles it owns in one process.
Do not add a storage lease, filesystem lock, fencing token, heartbeat, timeout-based takeover, deletion tombstone, quarantine protocol, or generic lock manager.
### 1.2 Read-only fork access may overlap the worker
The server owns repository administration and may fork a source while its Session worker continues writing it. The source side of that fork:
- opens the exact source container without creating it;
- is read-only;
- uses one deferred `BEGIN` transaction;
- reads the version-gated Session row, scalar values, and selected entries/branch index in that transaction;
- never upgrades to a write or claims writable authority;
- closes after `COMMIT`/`ROLLBACK`.
SQLite WAL permits later worker commits while the read transaction remains open. The fork sees each source commit entirely before or entirely after its snapshot boundary, never a mixture.
### 1.3 Same-repository fork ordering remains distinct
A source already open in the same repository uses its active `SqliteStorage.snapshot()` path. That snapshot is queued on the source `commitQueue`, preserving WP06's admitted-commit ordering seam and the existing conformance case.
Do not replace this path with an independent connection. Instead, bind active storage lookup to exact physical identity plus Session ID so metadata for another container cannot select it accidentally.
### 1.4 Destination ownership does not overlap
`SessionRepo.create()` and `fork()` continue returning an open Session. A server may temporarily own that new destination, capture its metadata, and close it before launching a worker. This is a valid ownership transfer, not a reason to redesign `SessionRepo` into a record-only API.
## 2. Problems in current source
### 2.1 SQLite duplicates host ownership
Current source contains:
- `writer_lease` schema state;
- claim, renew, and release helpers;
- lease claims in create/open/fork;
- an idle renewal timer and lease-loss path in `SqliteOpenSession`;
- a pre-commit renewal callback in `SqliteStorage`;
- lease-based deletion checks.
This is a second, incomplete ownership system. Its pre-commit renewal is not atomic with the following data transaction, but the correct fix is deletion of the ownership mechanism, not transaction-local fencing.
### 2.2 Non-creation access can create files
The database factory exposes only `open(path)`, which creates a missing SQLite file. Metadata open, listing probes, deletion, and fork-source reads must not turn a removed path into an empty database.
Fork-source reads also configure `PRAGMA journal_mode = WAL`, which is a write-oriented setup step and must not run on a read-only connection.
### 2.3 Delete does not reserve its local critical section
`delete()` checks `pendingIds` but does not reserve the ID. A same-repository create/open/fork destination can enter while asynchronous destructive work is in progress. Host lifecycle owns cross-process ordering; the repository still must serialize its own local operations.
### 2.4 Physical identity, paths, and close need correction
- `openStorages` is keyed only by Session ID, so the same ID at another physical path may select the wrong active source.
- `create()` makes `options.directory`, not the parent of an explicit `databasePath`.
- Per-session filenames interpolate arbitrary caller IDs directly; `/`, `\`, `..`, `%`, and platform separators must not escape `directory`.
- `repo.close()` uses fail-fast `Promise.all`, so it may return before every open Session has attempted to drain and close.
Deterministic list ordering, bind-variable limits, branch-copy cost, fork scalar filtering, prepared statements, and VACUUM policy remain separate.
## 3. Required result
### 3.1 Remove storage-layer writer ownership
Delete all runtime lease behavior:
- delete `src/sqlite/session/writer-lease.ts`;
- remove `writer_lease` from WIP `001_initial.sql`;
- remove lease deletion from `deleteSessionRows()`;
- remove claim/renew/release code from `SqliteSessionRepo`;
- remove `beforeCommit` from `SqliteStorage`;
- remove renewal/release options, timer, and `leaseError` from `SqliteOpenSession`;
- remove lease-specific tests and replace them with host-authority and live-fork coverage.
Keep:
- the Storage `commitQueue`;
- one `BEGIN IMMEDIATE` transaction per commit;
- in-transaction `next_seq` allocation;
- entry/usage uniqueness and parent triggers;
- Session mutation admission and close draining;
- process-local duplicate-open rejection.
Format 4 remains WIP. Remove the table from new schema in place; an old file containing an unused `writer_lease` table remains readable and the table and stale rows are ignored forever. Post-WP07 code does not delete them because doing so serves no runtime behavior. A pre-WP07 binary cannot open a new post-WP07 database without that table; backward compatibility for this WIP format is not required. Add no migration, compatibility path, or storage-version bump.
### 3.2 Add explicit database open modes
Extend `SqliteDatabaseFactory` with narrow operations:
- `open(path)` — intentional creation or create-if-missing;
- `openExisting(path)` — read-write open that fails if the file does not exist;
- `openReadOnly(path)` — read-only open that fails if the file does not exist.
The Node adapter uses `DatabaseSync(path, { readOnly: true })` for read-only access. Implement and test an actual no-create read-write mode for `openExisting`; do not rely on `access()` followed by a create-capable open.
Split connection setup:
- writable connections establish WAL mode and `busy_timeout`;
- read-only connections set only read-safe options such as `busy_timeout` and never attempt to change journal mode.
Use no-create modes for metadata open, listing probes, deletion, and fork-source reads.
### 3.3 Preserve both fork-source paths
**Source open in this repository:** retain `SqliteStorage.snapshot()` and queue it after prior admitted commits. Replace the ID-only active map key with canonical `(containerPath, sessionId)` identity and use the same helper for publish, lookup, and removal.
**Source not open in this repository:** this includes a closed source and a source currently owned by a worker in another process. Open the exact source through `openReadOnly`, then capture it in one deferred read transaction. Validate the Session row and storage version inside that transaction. Do not consult destination reservations, claim source ownership, or block the worker's later commits.
The destination remains a normal writable create/fork transaction after source capture. In shared-container mode the source worker and destination transaction may use the same file; SQLite serializes destination writes while preserving Session row isolation.
### 3.4 Make deletion locally exclusive
The host must close the Session worker before calling `repo.delete()`. Direct cross-process deletion of a live Session is unsupported.
Within one `SqliteSessionRepo`, deletion must reserve the Session ID from entry through completion and release it in `finally`:
- an already open or reserved Session rejects deletion;
- create/open/fork destination for that ID rejects while deletion runs;
- shared-container deletion removes only the target Session's rows in one `BEGIN IMMEDIATE` transaction on one connection;
- per-file deletion removes the database and its WAL/SHM sidecars after a no-create open/existence check;
- a missing Session rejects without creating a file.
Do not add a lease check, tombstone, quarantine rename, or stale-deleter protocol. Cross-process removal ordering is the server's responsibility.
### 3.5 Bind metadata to physical identity and make paths safe
- Canonical identity is `(canonical container path, sessionId)`.
- In per-file mode metadata must identify the repository-affine encoded path for its durable ID.
- In shared-container mode metadata must identify the configured canonical container and Session ID.
- A foreign or mismatched path must never alias a local active source. Pin in focused tests and the SQLite README whether foreign source metadata is rejected or read only from its exact path; do not silently substitute a local storage by ID.
- Create `dirname(databasePath)` when `databasePath` is configured.
- Encode arbitrary explicit IDs into safe per-session filenames without changing the durable ID. The encoding must prevent path escape and round-trip through metadata/list/open/fork.
- Two different Session IDs in one shared container remain independently addressable.
### 3.6 Drain all repository-owned closes
`SqliteSessionRepo.close(context)` must:
1. seal repository admission once;
2. start close on every currently open Session;
3. wait for every close to settle;
4. resolve when all succeed;
5. otherwise reject only after all cleanup attempts, returning the one error or an `AggregateError` containing all failures;
6. return the same promise on repeated close.
This is backend-local resource cleanup. Do not change the shared `SessionRepo` interface or JSONL lifecycle in this package.
## 4. Implementation slices
### Slice A — remove writer leases
Files:
- delete `src/sqlite/session/writer-lease.ts`;
- `src/sqlite/migrations/001_initial.sql`;
- `src/sqlite/session/session-row.ts`;
- `src/sqlite/storage.ts`;
- `src/sqlite/session.ts`;
- `src/sqlite/repo.ts`;
- lease-focused repository tests.
Tasks:
1. Remove schema/runtime lease state and timer behavior.
2. Preserve commit serialization, one write transaction, mutation draining, and process-local duplicate-open behavior.
3. Prove ordinary commits now use one write transaction rather than renewal plus write.
### Slice B — no-create opens and deletion reservation
Files:
- `src/index.ts`;
- `src/sqlite/types.ts`;
- `src/sqlite/repo.ts`;
- focused adapter/repository tests.
Tasks:
1. Add `openExisting` and `openReadOnly` with tested no-create behavior.
2. Separate writable and read-only connection configuration.
3. Reserve deletion locally for its full critical section.
4. Delete one shared-container Session in one transaction; preserve unrelated Sessions.
### Slice C — live-source read-only forks and identity
Files:
- `src/sqlite/repo.ts`;
- repository/conformance tests.
Tasks:
1. Key active storage by canonical container plus Session ID; keep same-repository queue ordering.
2. Use one independent read-only deferred transaction for non-open/live-worker sources.
3. Validate source metadata/version inside the snapshot.
4. Preserve shared-container destination behavior.
### Slice D — paths and close draining
Files:
- `src/sqlite/repo.ts`;
- focused repository/conformance tests.
Tasks:
1. Create the actual custom container parent.
2. Encode arbitrary IDs safely.
3. Reject or exactly handle foreign metadata without active-source aliasing.
4. Make repository close all-settled and error-complete.
### Slice E — documentation
Files:
- `packages/agent/docs/harness.md`;
- `packages/agent/docs/post-wp05-roadmap.md`;
- `packages/agent/docs/values.md`;
- `packages/session-backends/sqlite-node/README.md`;
- changelog only under normal branch rules.
Document host-owned writable authority, the two fork-source paths, no-create opens, local deletion reservation, and absence of storage-layer ownership.
## 5. Required tests
Use real independent `node:sqlite` connections. Test-only wrappers may expose deterministic transaction boundaries; production code gets no sleeps or race flags.
### Lease removal
- new schema has no `writer_lease` table;
- create/open/fork/commit/close perform no lease reads or writes and start no renewal timer;
- same-repository duplicate writable open still rejects through process-local reservation;
- ordinary commit remains one `BEGIN IMMEDIATE` transaction.
### Live fork source
For both per-file and shared-container layouts:
- a server repository forks a source held open by a separate repository/connection representing its worker;
- source capture uses a distinct read-only connection and claims no writable ownership;
- a source commit completed before the snapshot boundary appears wholly in the fork;
- a commit after the read snapshot is established can complete before the reader closes and is wholly absent from that fork;
- no fork contains an entry without the same commit's Branch tip/value/stats changes;
- a later fork includes the later commit;
- same-repository admitted-commit fork conformance remains unchanged.
### Deletion
- open/reserved Session first → same-repository delete rejects;
- delete reservation first → same-repository create/open/fork destination for that ID rejects;
- shared deletion removes only the target rows;
- per-file deletion removes database/WAL/SHM files;
- missing-path open/list/fork/delete does not create an empty database;
- tests state the host precondition: worker close precedes deletion; no cross-process bypass safety is promised.
### Identity and paths
- same Session ID at two physical paths cannot cross-select active source storage;
- two Session IDs in one shared container remain independent;
- `databasePath` succeeds when its parent does not exist;
- explicit IDs containing `../`, `/`, `\`, `%`, dots, and Unicode stay inside `directory` and preserve the metadata ID;
- metadata returned by create/list/open/fork names the actual container.
### Close
- one Session close failure does not prevent every other Session cleanup attempt;
- multiple failures are reported after all settle;
- repeated repository close returns the same promise;
- every successfully closed Session releases its connection.
### Regression
- existing storage and repository conformance remains semantically unchanged;
- fork destination reservation and same-repository source ordering remain intact;
- fork snapshots exclude operation/pending/result/usage/application state exactly as before;
- shared-container create/list/open/fork/delete remains supported;
- every write transaction still uses `BEGIN IMMEDIATE`;
- no migration, storage-version bump, compatibility layer, or replacement ownership primitive appears.
## 6. Validation and review
After each code slice:
```bash
npm run check
```
Run each modified focused test from `packages/session-backends/sqlite-node` with the repository Vitest binary. Final validation:
```bash
./test.sh
```
Review checkpoints:
1. Fable after Slice A: verify no storage ownership mechanism remains and close draining is intact.
2. Fable after Slice C: verify same-repository ordering and live-worker read-only overlap both hold.
3. Final Fable review of source, tests, docs, and exclusions.
Delegated reviews use provider `anthropic` and model `claude-fable-5`.
## 7. Exclusions
Do not include:
- any storage lease, lock, fence, heartbeat, takeover, tombstone, or quarantine;
- a record-only `SessionRepo` redesign or compatibility facade;
- server/router/worker-manager redesign; record any backend-independent lifecycle race separately;
- SQLite branch-segment redesign or uncompacted-divergence optimization;
- fork scalar filtering/indexing;
- `getEntries` bind-limit chunking or general query-limit normalization;
- statement caches or stats aggregation optimization;
- catalog redesign, async database replacement, or VACUUM policy;
- search/FTS;
- R11 migration machinery or a storage-version bump;
- [mobile assistant-output handoff](../mobile-handoff/01-harness/05-assistant-output/message-update.md) changes or JSONL compaction;
- repository-wide `SessionRepo.close()` contract changes;
- removal of shared-container support;
- transaction DSLs, schedulers, generic lock managers, or compatibility layers.
If implementation requires an excluded item, stop and revise the handoff rather than expanding silently.
## 8. Exit condition
WP07 is complete when:
- SQLite contains no active or schema-defined writer lease and implements no replacement ownership mechanism;
- host ownership is the documented single-writer authority;
- same-repository active-source forks retain their queued commit boundary;
- live worker-owned sources fork through an independent read-only snapshot while later WAL commits continue;
- deletion reserves its same-repository critical section and assumes worker-first host removal;
- non-creation paths cannot create empty databases;
- active source identity includes physical container plus Session ID;
- explicit IDs cannot escape the directory and custom database parents are created;
- repository close waits for every cleanup attempt;
- shared-container mode remains fully covered;
- focused tests, `npm run check`, and `./test.sh` pass;
- final Fable review reports no blocker.

240
third_party/pi/packages/agent/docs/work-packages/08-named-branch-streaming-forks.md vendored
View File

@@ -1,240 +0,0 @@
# WP08 — Named-branch and tree forks with streaming copies
**Status: in progress — implementing Slice A.**
This package replaces the fork contract: `ForkOptions` gains a mandatory scope and a mandatory named source branch, branch forks validate a complete configured source AgentLane and ancestry membership, tree forks copy the complete immutable tree plus current application values/lists, and all three backends replace materialized source snapshot arrays with bounded-memory streaming copies. A JSONL fork never repairs or mutates its source. One closed core classifier owns every namespace's fork disposition.
WP07 is a hard dependency and is preserved unchanged: no-create database modes, canonical `(containerPath, sessionId)` identity, independent read-only WAL reader for external/live-worker sources, repository-local deletion reservation, and all-settled close. This package supersedes only the roadmap's "SQLite fork cost" performance item.
## 0. Mandatory reading
Read completely before editing:
1. `packages/agent/docs/harness.md` §§0.6, 1.3–1.7, 2.3, 2.7–2.9, Part 9 (invariants 3–5, 13, 16; ledger completeness).
2. `packages/agent/docs/values.md`, especially "Forks and rewrites" and the backend sections.
3. `packages/agent/docs/post-wp05-roadmap.md` (SQLite fork cost, repository lifecycle context).
4. Completed WP06 §7 and WP07 (historical; do not edit them).
5. `packages/agent/src/harness/session/fork.ts`, `types.ts` (`ForkOptions`, `SessionRepo`), `values.ts`, `in-memory-storage-state.ts`, `memory.ts`, `session/index.ts`.
6. `packages/agent/src/harness/session/jsonl/repo.ts`, `jsonl/storage.ts`, `jsonl/codec.ts`, `jsonl/legacy-v3.ts`, `jsonl/types.ts`.
7. `packages/session-backends/sqlite-node/src/sqlite/repo.ts`, `storage.ts`, `session/values.ts`, `session/entries.ts`, `session/branch-entries.ts`, `types.ts`.
8. `packages/agent/src/harness/session/testing/conformance/session-repo.ts` and every test named in §5.
9. `packages/agent/src/harness/session/testing/benchmark/session-repo.ts` and both `session-repo.bench.ts` files.
Do not use `dist/` output as implementation input. WP00–WP07 documents and released changelog sections are immutable in this package.
## 1. Fixed architecture
### 1.1 Public contract
```ts
export type ForkOptions =
| { scope: "branch"; branch: string; entryId?: string; position?: "before" | "at"; id?: string }
| { scope: "tree"; id?: string };
```
`scope` is required. `branch` is required for branch scope. There is no default scope, no implicit `main`, and no compatibility alias.
**Branch scope** requires a complete configured source AgentLane: `pi.branch.tip/{branch}`, `pi.lane.config/{branch}`, and `pi.lane.state/{branch}` must all exist. A missing tip rejects (`unknown branch`); a data-only Branch (tip without config/state) rejects; a partial config/state pair, or `laneConfig`/`laneState` present without a `branchTip`, is corruption and rejects. `entryId`, when supplied, must be an entry on that Branch's current tip ancestry (inclusive); an entry elsewhere in the tree rejects. Omitted `entryId` means the current tip. `position` defaults to `"at"`; `"before"` selects the target's parent and may yield a `null` destination tip (before the root entry, or a `null` source tip with no `entryId`) — legal. The destination contains exactly one Branch with the same name, the selected tip, the copied `LaneConfiguration`, and fresh idle lane state `{ currentOperationId: null, lastOperationId: null, inbox: [] }`. No other Branch or lane exists in the destination.
This deliberately makes branch forks of data-only Branches — including legacy v3 imports whose main lacked reconstructable model/thinking history — unavailable. Tree scope remains available for those sources.
**Tree scope** copies: every immutable entry, including entries unreachable from every current Branch tip; every Branch tip verbatim; every configured lane's config plus fresh idle lane state under the same name; data-only Branches (tip present, config/state absent) as data-only Branches. A partial config/state pair, or `laneConfig`/`laneState` without a `branchTip`, is corruption in both scopes: tree scope **rejects** it rather than dropping it.
**Both scopes**: copy `pi.session.name`; copy `pi.entry.label` values only for copied entries; exclude the usage ledger, `pi.result`, all `pi.op.*`, all `pi.pending.*` (entries, tool checkpoints, assistant frames), and every trace of open-operation state. Destination usage totals start at zero; destination `messageCount` equals the number of copied message entries, exactly as current conformance already proves. Destination metadata records `parentSessionId = source.id`. Copied entries, values, and list elements preserve their original `seq`; the destination `nextSeq` high-water mark is the source's, so no sequence is reused. Transformed built-ins keep coherent seqs: the rewritten `branchTip` and each fresh idle `laneState` reuse the seq of the source's current corresponding value row, and copied config, name, labels, application values, and list elements retain theirs — destination writes stay in original seq order under an unchanged high-water mark.
**Application values/lists (outside the reserved `pi`/`pi.*` namespaces)**: tree scope copies every current scalar value and every surviving list element with original seqs; branch scope copies none. A `seq <= tipSeq` cutoff is forbidden as "historical reconstruction" — it is provably not one:
```text
Scalar: TX[seq 10: set my-app.state = v1] · TX[seq 12: insert e1] ·
TX[seq 50: set my-app.state = v2] (v2 describes work after e1)
Branch fork at e1, cutoff seq <= 12: only v2 exists (v1 was replaced, no
history is retained); 50 > 12 excludes it → state absent, though the app
demonstrably had state v1 at the fork point. The cutoff cannot recover v1.
List: append seq 5 · append seq 20 · deleteList seq 40 · append seq 60
Cutoff seq <= 30: elements 5 and 20 no longer exist (whole-list delete
destroyed them); only 60 survives and is excluded → empty list, though the
list held {5, 20} at seq-30 time.
```
Any cutoff filters *survivors*, silently mixing rewound intent with post-deletion reality. Branch scope therefore copies nothing application-owned, matching its fresh idle lane state and zero ledger; applications own their own re-derivation.
### 1.2 One closed fork classifier
All namespace fork knowledge lives in one core module beside `session/values.ts` and `session/fork.ts` (e.g. `session/fork-policy.ts`). It is a closed switch, not a registry, plugin policy, or DSL:
- `pi.op.*`, `pi.pending.*`, `pi.result` → exclude, both scopes.
- `pi.session.name` → copy.
- `pi.entry.label` → copy iff the keyed entry is copied.
- `pi.branch.tip`, `pi.lane.config`, `pi.lane.state` → structured lane actions; the scope-specific rules (branch keeps only the named lane and rewrites its tip; lane state is always replaced with fresh idle state; tree keeps all) live in one shared driver consumed by all backends.
- The exact namespace `pi` and any other `pi.*` namespace → the fork **fails**, but only when current surviving state exists at fork time (a current scalar row or a surviving list element). Historical writes later replaced or deleted are absent from current state on every backend and must not alone fail a JSONL fork — behavior is backend-equivalent. Introducing a new built-in namespace without declaring its fork semantics must break fork tests, not silently copy or drop state.
- Neither `pi` nor `pi.*` → application: copy on tree, exclude on branch. The only built-in list namespace (`pi.pending.assistant_frame`) excludes; application lists follow the application rule.
The driver exposes a streaming shape — accept one committed value/list write (or current row), emit zero or more destination writes, `finish()` emits fresh idle lane states and the rewritten branch tip — holding only bounded state (lane names, destination tip). Entry-copy membership is a backend-supplied predicate so each backend uses its own index. `createForkSnapshot`, `forkSnapshotWrites`, `ForkSourceSnapshot`, `ForkDestinationSnapshot`, and the `entriesComplete` escape hatch are deleted.
### 1.3 Bounded-memory backend procedures
Auxiliary memory during a fork must stay bounded regardless of source size. The unavoidable exceptions are Memory's destination state and the returned open destination Session (JSONL destination state after open). Source copy paths must not call array-returning full reads: no `snapshotEntriesAndValues()`, `captureForkSource()`, `readAllScalarValueRows()`, `readAllEntryRows()`, whole-file `readTextFile`, or SQLite `.all()` over unbounded row sets.
**Memory.** At a source `commitQueue` boundary, build the destination `InMemoryStorageState` directly by iterating source maps once through the classifier/driver. Branch scope walks `parentId` from the branch tip toward the root to compute the ancestry id set and verify `entryId` membership; copy exactly that set. No intermediate snapshot arrays; `MemoryStorage.fromSnapshot` and `captureForkSource` are removed.
**JSONL.** The source is read through a read-only path that captures a fixed file prefix — open a read handle, record the current length at capture time, and read only that prefix — and never writes the source: no torn-tail truncation/rewrite, no v3 normalization persistence, no `.tmp` beside the source. A torn or incomplete final line inside the captured prefix is discarded in memory. The source `nextSeq` high-water mark is derived as `max(header.nextSeq when present, highest complete captured write seq + 1)` — ordinary headers are not rewritten on every append, and a torn incomplete final write never advances it.
- *Both scopes* use two passes over the captured prefix with one temporary disk-backed index (deleted in `finally` on success and failure). Pass 1 streams every line and folds value/list writes into the index: per scalar address, the seq of the current surviving `set` (absent after a trailing `delete`); per list address, the set of surviving element seqs (whole-list delete clears it); plus entry `id → parentId` (parents always precede children) where branch scope needs ancestry. The fold applies the same current-state semantics replay does, without whole-state RAM.
- Pass 2 streams the prefix again in order and emits a write only when the index proves it current and the classifier selects it: an entry write in the selected set; a scalar `set` whose seq equals the index's current-row seq for its address; a list `append` whose element seq is in the survivor set. Deletes, superseded sets, and dead application value/list history are never emitted as destination state — the destination file contains only current selected rows, already in original seq order because source lines are seq-ordered.
- Transformed built-ins emit in place: when pass 2 reaches the seq of the source's current `branchTip` row for a kept branch it emits the (possibly rewritten) tip under that same seq, and when it reaches the seq of the current `laneState` row for a kept configured lane it emits fresh idle lane state under that same seq. No tail appends exist, so replay stays seq-monotonic and the high-water mark is untouched.
- *Branch scope* walks tip→root through the disk index to materialize the ancestry set on disk and verify `entryId` membership before pass 2; labels emit only for member entries.
- The destination is staged in a temp file and atomically renamed (existing `publishFileAtomically`); failure removes the temp file and the index. Legacy v3 splits by source state. Forking an **open** legacy-v3 `JsonlStorage` **rejects with a clear error** until a normal non-empty commit has upgraded and persisted its normalized format-4 ids: v3 normalization mints nondeterministic UUIDv7 tails, so an independent disk reparse cannot reproduce the ids the open Session exposes or validate a caller `entryId`; the fork must not mutate/upgrade the source itself. A **closed** legacy-v3 source uses the bounded fixed-prefix disk-backed parser/normalizer and never touches the file: tree forks succeed; branch forks may use an omitted `entryId` (the default normalized tip) when the import reconstructs a complete configured lane, while any caller-supplied process-local id from an earlier open is not stable across reparses and rejects normally; a data-only reconstructed lane rejects as everywhere. Ordinary writable `JsonlStorage.open` may retain in-memory normalization, but resident state is never a fork source.
- There is **no resident-state path**: an open same-repository JSONL source enqueues a short boundary callback on the source `commitQueue` whose only job is to capture the fixed read-only file-handle prefix (open handle, record length), then releases the queue; the fork runs the same two-pass disk-backed procedure over that prefix while later source appends proceed. Committed writes are durable in the file before they are applied to resident state, so the prefix is authoritative at the boundary, and file order preserves seq order without resident-Map ordering ambiguity.
**SQLite.** Iterator-based row transfer through a bounded-memory temporary on-disk SQLite **staging database**, used uniformly for per-file and shared-container layouts. The source reader never streams directly into the destination transaction — a WAL reader and one writer can coexist, but in shared-container mode the destination writer would hold the container's sole write lock and block the post-boundary source writer while capture is still streaming; staging into a separate file removes that coupling. Use prepared-statement iteration (`iterate`/stepwise), never source-sized `.all()` snapshot arrays:
- *External/closed/live-worker source:* WP07 path — `openReadOnly` on the exact canonical path, one deferred read transaction, session row and storage version validated inside it.
- *Same-repository open source:* open the independent read-only connection **first**, then enqueue a short boundary callback on the source Storage `commitQueue` whose only job is to `BEGIN` and establish the independent reader's snapshot (issue a trivial read) before releasing the queue. Do not begin a read transaction on the source writer connection and then release its queue, and do not hold the queue for the copy duration.
- *Stage:* while the source reader remains open, stream selected entries `ORDER BY seq` and classifier-selected current scalar/list rows (SQL-level namespace prefilters matching the classifier — enumerated built-in namespaces plus the application predicate excluding exact `pi` and `pi.%` — with each row still passing the classifier) into the temporary staging database in bounded batches. Branch scope enumerates the ancestry through the `branch_entries` segment chain up to the selected entry and answers `entryId` and label membership through that index, not an in-RAM id set. Later source commits use the original writer connection and may complete while staging streams, in **both** layouts, because stage writes target another file.
- *Publish:* close/commit the source read transaction, then stream the stage into one destination `BEGIN IMMEDIATE` transaction — entries in seq order maintaining the destination branch index and `message_count` incrementally, then values/list elements — and delete the staging database in `finally` on success and failure. Destination `next_seq` is the source's. Shared-container destinations write only the new session's rows.
### 1.4 Preserved WP07 behavior
Destination id reservation across create/open/fork/delete, no-create opens, foreign-metadata rejection, per-file and shared-container layouts, WAL commit-boundary wholeness (a source commit is wholly inside or wholly outside one fork), and all-settled repository close are unchanged.
### 1.5 Coding-agent status (record only)
`/fork`, `/clone`, and `--fork` run entirely on the legacy `SessionManager` (`createBranchedSession`, `forkFrom`) and are not migrated here. Future mapping when coding-agent adopts `SessionRepo`: `/fork` → `{ scope: "branch", branch: "main", entryId, position: "before" }`; `/clone` → `{ scope: "branch", branch: "main" }`; `--fork` → `{ scope: "tree" }`.
## 2. Problems in current source
- `ForkOptions` defaults `scope` to `"branch"` and hard-codes source/destination `main` (`fork.ts`, both backend branch readers).
- No ancestry membership check: `selectForkContents` walks parents from any supplied `entryId` anywhere in the tree and labels the result `main`.
- Every backend materializes the full source: `snapshotEntriesAndValues()` (Memory/JSONL), `readAllScalarValueRows` + `readAllEntryRows`/`scanBranchEntries` arrays (SQLite), all funneled through in-memory `createForkSnapshot`.
- `createForkSnapshot` renumbers copied scalar values with fresh seqs and computes `nextSeq` as `max entry seq + 1`, losing original value seqs.
- A JSONL fork of a closed source uses `JsonlStorage.open()`, which rewrites the source file on a torn tail — a fork can mutate its source today.
- Lists are never copied; application values are excluded with no declared tree policy (`values.md` explicitly defers it).
- Namespace fork knowledge is repeated across `fork.ts`, `values.md` prose, and conformance assertions; nothing forces a new `pi.*` namespace to declare fork semantics.
- `entriesComplete?: false` exists only to let SQLite branch snapshots skip tip validation for other branches.
## 3. Required result
1. `ForkOptions` and validation exactly as §1.1; all rejection paths create no destination file, database rows, or reserved-but-leaked ids.
2. The closed classifier/driver of §1.2, exported from core and consumed by all three backends; unknown reserved namespaces (exact `pi` or undeclared `pi.*`) fail the fork only when current surviving state exists, identically on every backend.
3. Streaming procedures of §1.3 on all three backends; `createForkSnapshot`/`captureForkSource`/`snapshot()` fork plumbing and their exports removed from `session/index.ts` and the sqlite-node import surface.
4. Seq preservation (including reused seqs for rewritten tips and fresh idle lane states) and source `nextSeq` high-water on every backend (JSONL derives it per §1.3); identical logical destination state across backends for identical sources (conformance).
5. JSONL source non-mutation, including torn-tail sources and legacy v3 sources.
6. Documentation: `harness.md` §2.7 (and the §1.7 fork-related sentences), `values.md` "Forks and rewrites" and backend snapshot mentions, `post-wp05-roadmap.md` (retire the SQLite fork-cost item, add/point to this package), `packages/session-backends/sqlite-node/README.md` fork paragraphs, `packages/agent/benchmark/session/README.md` if dataset wording changes. Historical WP docs and released changelogs untouched. Format/storage versions unchanged; no migration.
## 4. Implementation slices
### Slice A — contract and classifier (core, Memory reference)
Files: `session/types.ts`, new `session/fork-policy.ts`, `session/fork.ts` (rewritten or deleted), `session/values.ts` (only if new helpers are needed), `session/index.ts`, `session/memory.ts`, `session/in-memory-storage-state.ts`, conformance `testing/conformance/session-repo.ts`, `test/harness/memory-conformance.test.ts`, `test/harness/memory-session-repo.test.ts`.
1. Replace `ForkOptions`; implement validation and the classifier/driver.
2. Rewrite Memory fork as direct destination construction at a commit-queue boundary.
3. Rewrite the shared fork conformance for the new contract (§5) and pass it on Memory.
### Slice B — JSONL streaming
Files: `session/jsonl/storage.ts`, `session/jsonl/repo.ts`, `session/jsonl/types.ts`, streaming reader support in the `FileSystem` capability (`harness/types.ts`, `harness/env/*`) if required, `test/harness/jsonl-session-repo.test.ts`, `jsonl-session-repo-conformance.test.ts`, `jsonl-storage.test.ts`, `jsonl-v3-migration.test.ts`.
1. Read-only fixed-prefix source capture with in-memory torn-tail discard; the same capture at the source `commitQueue` boundary for open sources; no source writes on any fork path.
2. Two-pass disk-backed current-state fold and emission for both scopes; branch ancestry membership through the disk index; in-place transformed built-ins; atomic destination publish.
3. Open-v3 fork rejection and the closed-v3 disk-backed parser/normalizer path; update v3 fork tests per §5.
### Slice C — SQLite streaming
Files: `sqlite/repo.ts`, `sqlite/storage.ts`, `sqlite/session/values.ts`, `sqlite/session/entries.ts`, `sqlite/session/branch-entries.ts`, `sqlite/types.ts`, `test/repo.test.ts`, `test/repo-conformance.test.ts`.
1. Replace `SqliteStorage.snapshot()`/array snapshot helpers with the independent-reader-plus-boundary-callback design and iterator transfer into the temporary staging database, then stage-to-destination publication with `finally` cleanup.
2. Branch ancestry/membership/labels through the branch index; classifier-matching SQL prefilters.
3. Preserve WP07 identity, reservation, no-create, and close coverage; both layouts.
### Slice D — benchmarks and documentation
Files: `testing/benchmark/session-repo.ts`, both `session-repo.bench.ts` files, `benchmark/session/README.md`, `harness.md`, `values.md`, `post-wp05-roadmap.md`, sqlite-node `README.md`, changelogs only under normal branch rules.
Update fork option literals, add large-source fork benchmarks (tree and branch), and land the §3.6 documentation set.
## 5. Required tests
### Contract and validation
- branch scope rejects: unknown branch name; data-only Branch (tip only); partial config/state pair (corruption); `entryId` not on the named Branch's tip ancestry (present elsewhere in the tree); `entryId` unknown; `entryId` with a `null` tip. Each rejection leaves no destination artifact and releases its reserved id.
- branch scope accepts: omitted `entryId` (tip), explicit tip, mid-ancestry entry, `position: "before"` at a mid entry and at the root entry (`null` destination tip), `null` source tip with no `entryId`.
- destination shape: exactly one Branch, same name, copied config, fresh idle lane state, no other lane values, `parentSessionId` set.
- tree scope: unreachable entries copied; every tip copied; configured lanes get config plus fresh idle state; data-only Branches stay data-only; partial pairs fault.
- both scopes: session name copied; labels only for copied entries (branch fork excludes labels of non-ancestry entries); `pi.result`, `pi.op.*`, `pi.pending.*` (pending entries, tool checkpoints, assistant frame lists), and usage rows absent; exact stats expectation — a fork copying N message entries reports `getStats()` = `{ messageCount: N, usage: all-zero }`, matching current conformance; the ledger-completeness invariant ("a fork's ledger starts at zero") retained.
### Application values and lists
- tree fork copies every non-`pi.*` scalar and every surviving list element with original seqs; list cursors from the source page identically in the destination; a list deleted-then-reappended in the source reproduces only survivors.
- branch fork copies no application values/lists (pin the §1.1 scalar and list traces as regression cases: post-fork-point overwrite and delete-destroyed elements must not resurface under any implementation).
- current surviving state in an unknown reserved namespace (exact `pi` or an undeclared `pi.*` scalar or surviving list element) fails the fork on every backend; the same namespace **set then deleted** before the fork fails no backend — including JSONL, where the dead history remains as physical lines and the disk fold must classify it as absent. Construct both cases via raw committed writes.
### Sequence preservation
- copied entries, values, and list elements keep source seqs; rewritten branch tips and fresh idle lane states occupy the seqs of the source's current corresponding rows; destination `nextSeq` equals the source high-water mark and the first post-fork commit allocates above it on every backend (JSONL header `nextSeq`, SQLite `next_seq`).
- JSONL derives the high-water mark as `max(header.nextSeq when present, highest complete captured write seq + 1)`; a torn incomplete final write does not advance it.
### Bounded memory and source non-mutation
- instrumented source readers (test decorators/spies) assert fork paths never call `snapshotEntriesAndValues`, `captureForkSource`, `readAllScalarValueRows`, `readAllEntryRows`, whole-file `readTextFile`, or source-sized `.all()` arrays on entries/values/lists — bounded iterator steps and staging-database operations are permitted; deterministic large fixtures (extend the existing benchmark dataset generators) exercise tree and branch forks over multi-thousand-entry sources.
- SQLite staging databases are removed on success and failure; a failed fork leaves neither a stage file nor destination rows.
- JSONL: fork of a closed torn-tail source succeeds, the destination excludes the torn transaction wholly, and the source file bytes are unchanged (byte-compare before/after); fork of a legacy v3 source leaves the source file unchanged; no `.tmp` appears beside the source; temporary ancestry index files are removed on success and failure.
- JSONL fixed-prefix capture: a source append completed after capture is wholly absent from the fork.
- JSONL destination content: the destination file contains only current selected scalar rows and surviving list elements in original seq order — no delete records, superseded sets, or dead application history; the rewritten tip and fresh idle lane state sit at the source current rows' seqs.
- legacy v3: forking an **open** v3 source rejects with the clear pre-upgrade error; after one normal non-empty commit upgrades it to format 4, a fork succeeds and preserves the persisted ids; a **closed** v3 tree fork succeeds through the disk-backed parser (instrumented: no whole-state in-memory normalization on the fork path) and leaves the source byte-identical; a closed v3 branch fork with omitted `entryId` succeeds when a complete configured lane is reconstructed, a caller-supplied id from an earlier open rejects, and a data-only reconstructed main rejects.
### Coordination and ordering
- Memory open-source forks keep the queue-boundary conformance case: a commit admitted before the fork appears wholly; one admitted after does not.
- JSONL open-source forks capture the fixed prefix at the `commitQueue` boundary: a commit admitted before the boundary appears wholly; a source append completed after boundary capture proceeds while the fork is still streaming and is wholly absent from that fork.
- SQLite same-repo: the independent reader opens first, the boundary callback on the source `commitQueue` establishes its snapshot and releases the queue, and a subsequent source commit on the writer connection **completes while the reader is still streaming into the stage** and is wholly absent from that fork; a later fork includes it wholly. Assert this in **both** per-file and shared-container layouts (satisfiable in shared containers because stage writes target another file), plus the WP07 live external-worker case unchanged.
- destination-reservation races (create vs fork on one id, both orders) unchanged.
### Backend equivalence
- one shared conformance source (entries, unreachable entries, multiple lanes, data-only Branch, labels, app scalars/lists with deletes, open operation with pending/frame/checkpoint state, usage rows) forks to identical logical destination state on Memory, JSONL, and SQLite for both scopes.
## 6. Validation and review
After each slice: `npm run check`, then the modified focused tests via the repository Vitest binary from the owning package; `./test.sh` before final review. Grep guards: no remaining `createForkSnapshot|captureForkSource|ForkSourceSnapshot|entriesComplete` outside historical docs and this handoff; no **production fork source path** referencing `"main"` as a literal (this document and the recorded coding-agent mapping in §1.5 intentionally mention main).
Review checkpoints (delegated reviews use provider `anthropic`, model `claude-fable-5`):
1. After Slice A: contract, classifier closure, conformance shape.
2. After Slice C: same-repo ordering, reader independence, bounded-memory evidence.
3. Final review of source, tests, docs, and exclusions.
## 7. Exclusions
Do not include:
- compatibility aliases, a default scope, or implicit `main`;
- any `seq <= tip` cutoff or other historical-state reconstruction;
- a fork-policy registry, plugin hook, or DSL; per-address application opt-ins;
- copying usage rows, `pi.result`, or any open-operation state under any option;
- coding-agent `/fork`, `/clone`, `--fork`, or RPC migration;
- storage-version bumps, migrations, or format changes (formats stay WIP-in-place);
- J1 snapshot compaction (the streaming reader may be built shareable, nothing more);
- SQLite branch-segment redesign or the uncompacted-divergence fix;
- `SessionRepo` interface changes beyond `ForkOptions` (repository `close()` belongs to the lifecycle package);
- precise-rewrite tooling; search; edits to WP00–WP07 documents or released changelog sections.
If implementation requires an excluded item, stop and revise the handoff.
## 8. Exit condition
WP08 is complete when:
- `ForkOptions` is exactly the §1.1 union with the stated validation, on all three backends;
- branch forks require a complete configured source AgentLane and enforce ancestry membership; tree forks copy the complete immutable tree, every tip, configured and data-only Branches;
- tree forks carry all current application values and surviving list elements with original seqs; branch forks carry none;
- one closed core classifier owns every namespace disposition; unknown reserved namespaces (exact `pi` or undeclared `pi.*`) fail forks exactly when current surviving state exists, equivalently on all backends;
- all supported fork paths — including closed legacy v3 sources — are streaming and bounded in auxiliary memory per §1.3, proven by instrumented-reader tests over large fixtures; forking an open legacy-v3 source is explicitly unsupported (clear rejection) until an ordinary commit upgrades it, not handled by an in-memory exception;
- JSONL forks never mutate their source, including torn-tail and legacy v3 sources, and JSONL destinations contain only current selected rows in original seq order;
- SQLite forks stage through a temporary on-disk database (source reader → stage while open, stage → one destination `BEGIN IMMEDIATE` after reader close, stage deleted in `finally`), with the independent-reader boundary design and concurrent later writer commits proven in both layouts, preserving all WP07 behavior;
- copied seqs and `nextSeq` high-water marks are preserved and backends produce identical logical destinations;
- shared conformance, focused backend tests, benchmarks, `npm run check`, and `./test.sh` pass;
- normative docs and the roadmap reflect the new contract with historical documents untouched;
- final Fable review reports no blocker.

517
third_party/pi/packages/agent/docs/work-packages/09-lane-snapshot-settled-tools.md vendored
View File

@@ -1,517 +0,0 @@
# Work package 09 — LaneSnapshot settled-but-unplaced tools
## Status and baseline
- Repository: `earendil-works/pi`
- Branch at handoff creation: `dev`
- Baseline commit: `d14d6b22327d545d6a253f932165b63e48d7f9c8`
- The user reported the worktree clean immediately before this handoff.
- This document began as an implementation handoff after conversation compaction and now records the implemented design. It is not itself the normative harness specification; `packages/agent/docs/harness.md` remains normative.
## Goal
Keep every started or settled-but-unplaced tool call visible in `LaneSnapshot.operation.runningTools` until its immutable `toolResult` entry is placed in the transcript.
The intended projection is:
```text
planned → not yet represented in runningTools
effect_pending → runningTools(status: "running")
outcome_ready → runningTools(status: "settled")
completed → transcript toolResult entry
```
For each call after it becomes presentation-active, `runningTools` and placed transcript entries must not have a gap or overlap. Placement is a source-prefix flush, not an all-tools barrier. Remove a call from `runningTools` on its own `entry_added`, never on `turn_end`.
## Original bug
A call disappeared between real-effect completion and source-ordered tree placement:
1. `packages/agent/src/harness/runtime/reducer.ts`: `tool_end` spliced the call out of `runningTools`.
2. `packages/agent/src/harness/runtime/lane.ts`: `captureLaneSnapshot()`, `case "tools"`, projected only `effect_pending` calls and skipped `outcome_ready` calls.
3. The finalized result was already durable at `pendingEntry(resultEntryId)`, but a fresh/reconnected snapshot could not display it.
4. It reappeared only after `entry_added` placed the immutable `toolResult` entry.
For a parallel batch `[A, B, C]`, if B settles while A remains pending, B may stay `outcome_ready` until A is ready. If A is already placed, B can place without waiting for C. Therefore clearing everything at `turn_end` is wrong: early-placed results would temporarily exist in both `transcript` and `runningTools`.
## Confirmed current architecture
Mini does **not** replicate structural object deltas.
- `packages/coding-agent/src/experimental/mini/worker/lane-service.ts` sends one initial full snapshot and then forwards individual `HarnessEvent` objects.
- `packages/coding-agent/src/experimental/mini/tui/session.ts` folds those events through `reduceLaneSnapshot()`.
- Reconnect/rebase fetches another complete snapshot.
- `tool_update` currently carries a complete replacement progress result, not a nested diff.
Implemented durable tool flow in `packages/agent/src/harness/runtime/drive/tools.ts`:
```text
prepare
→ before_tool
→ intent commit (effect_pending + effective args), then tool_start
→ execute/update/checkpoint
→ after_tool
→ finalize
→ publishToolOutcome staging commit (pendingEntry + outcome_ready), then tool_end
→ materializeReady prefix placement
→ entry_added
```
All real, immediate synthetic, cancellation, and recovery outcomes converge through `publishToolOutcome()`.
`Lane.settleOperation()` supports commit-bound events. It commits, publishes process-local state, constructs the event batch, and the public operation awaits delivery. In parallel execution, `materializeReady()` is scheduled only after the outcome-completion promise resolves. Consequently, `tool_end` is delivered after staging and before placement.
## Agreed event contract change
Do **not** add `tool_result_ready` or `tool_outcome_ready`.
Instead, redefine harness `tool_start`/`tool_end` as tool-call processing/result lifecycle events rather than exclusively real external-effect lifecycle events.
### Fresh executed call
```text
intent commit
→ tool_start
→ tool_update*
→ execute/finalize
→ TX[pendingEntry + outcome_ready + cleanup]
→ tool_end
→ source-ordered placement
→ entry_added
```
### Fresh synthetic call
```text
TX[pendingEntry + outcome_ready]
→ tool_start
→ tool_end
→ source-ordered placement
→ entry_added
```
The staging transaction's post-commit event batch contains `tool_start` followed by `tool_end`, so a watcher cannot observe either lifecycle event without the authoritative staged state.
Fresh synthetic calls include:
- unknown tool;
- argument preparation or validation failure;
- `before_tool` denial or invalid replacement arguments;
- genuine assistant `length`/truncated call handling;
- cancellation while still `planned` or after intent but before effect admission.
### Recovery
Historical lifecycle events are not replayed.
- A restored `effect_pending` call is already represented by the initial snapshot.
- Safe replay emits recovery-tagged `tool_start` from the checkpoint-clear commit and `tool_end` from the later outcome-staging commit.
- Unsafe interruption synthesis may emit a recovery-tagged `tool_end` without a newly emitted `tool_start`; the initial snapshot supplied the running row.
- A call already restored as `outcome_ready` appears as settled in the initial snapshot and needs no replayed end event before placement.
### Meaning of `tool_end`
After this change, `tool_end` means:
> The complete final tool result is durably staged and the call is `outcome_ready`.
It becomes the authoritative reducer transition from running to settled. It must be emitted **after** the staging commit, not before it.
The old distinction between actually executed and synthetic results was encoded by omitting lifecycle events. There is no in-repo runtime consumer requiring that distinction. If preserving it is desired, discuss adding an explicit field such as `execution: "executed" | "synthetic"`; this field was discussed but **not agreed**, so do not add it silently.
## LaneSnapshot type
Change `LaneSnapshot.operation.runningTools` in `packages/agent/src/harness/agent-harness.ts` to use one `result` field for both progress and final output. Do not retain a separate `partialResult` field in the snapshot.
Prefer a discriminated union so invalid combinations are unrepresentable:
```ts
type SnapshotTool =
| {
status: "running";
toolCallId: string;
toolName: string;
args: unknown;
result?: AgentToolResult<unknown>; // latest complete progress snapshot
}
| {
status: "settled";
toolCallId: string;
toolName: string;
args: unknown;
result: AgentToolResult<unknown>; // complete finalized result
isError: boolean;
};
```
The user explicitly agreed to the discriminated union.
`tool_update.partialResult` may remain named `partialResult` in the event API; the reducer assigns it to the snapshot row's unified `result` field.
The current mini transport sends semantic events, not structural deltas, so `tool_end` still carries the complete final result even if it equals the latest update. Unifying the snapshot field is still the correct state model.
## Exact reducer behavior
File: `packages/agent/src/harness/runtime/reducer.ts`
### `tool_start`
- Resolve the operation with `matchingOperation(snapshot, event.runId)`.
- Upsert by `toolCallId`; do not blindly push.
- Set `status: "running"`, `toolName`, `args`.
- Clear stale settled-only fields if replacing an existing row.
- Preserve no stale final result. A newly started/replayed call may receive later `tool_update` values.
Upsert is required because a watch may capture durable `effect_pending` state before the buffered `tool_start` event is delivered.
### `tool_update`
- Use `matchingOperation(snapshot, event.runId)`, not `snapshot.operation` directly.
- Find the matching row.
- For a running row, replace `result` with `event.partialResult`.
- Ignore wrong-operation or missing rows.
### `tool_end`
- Resolve with `matchingOperation(snapshot, event.runId)`.
- Find the existing row by batch-local `toolCallId`; only one tool batch is presentation-active at a time.
- Replace it with `status: "settled"`, preserving its arguments and using `event.result` and `event.isError`.
- This naturally removes the provisional interpretation of the old `result`; there is no separate `partialResult` to delete.
- The finalized result remains displayed until placement.
`tool_end` does not carry arguments and cannot create a row. Fresh synthetic `tool_start` and `tool_end` are emitted together after the staging commit, eliminating the old capture/event gap. Unsafe recovery relies on the initial snapshot's running row.
### `entry_added`
If `event.entry` is a message whose role is `toolResult`, remove the matching batch-local `toolCallId` from `snapshot.operation?.runningTools`, then apply the transcript update. Harness events are serialized, trusted, emitted exactly once, and not historically replayed, so neither duplicate-entry handling nor cross-batch identity is needed.
Do not clear tool rows on `turn_end`.
## Exact authoritative capture behavior
File: `packages/agent/src/harness/runtime/lane.ts`, `captureLaneSnapshot()`, `case "tools"`.
The assistant entry is already loaded once. For each batch call:
### `planned`
Skip. It has not become presentation-active yet.
### `completed`
Skip. Its `toolResult` entry must already be in the captured transcript.
### `effect_pending`
- Validate that `assistant.message.content[sourceIndex]` is the matching `toolCall` block.
- Read `operationToolArgs(operationId, turnId, sourceIndex)`; it is required for effect-pending calls.
- Read optional `pendingToolOutput(operationId, resultEntryId)`.
- Project:
```ts
{
status: "running",
toolCallId: block.id,
toolName: block.name,
args: persistedArgs,
...(checkpoint === undefined ? {} : { result: checkpoint })
}
```
### `outcome_ready`
- Validate the source tool-call block.
- Read `pendingEntry(call.resultEntryId)`.
- Require a message payload with role `toolResult`.
- Validate staged `toolCallId` and `toolName` against the source block.
- Read `operationToolArgs(...)` when present.
- Use `persistedArgs ?? block.arguments`. Immediate synthetic calls may never have written `operationToolArgs`, and this absence is legal only for the outcome-ready projection.
- Reconstruct the canonical `AgentToolResult` from the staged `ToolResultMessage` and the durable call termination flag as needed.
- Project `status: "settled"`, `result`, and `isError`.
The event and capture representations must normalize the final result identically so folding through `tool_end` equals a later authoritative snapshot. Pay attention to optional `details`, `usage`, `addedToolNames`, and `terminate`; do not rely on incidental object-property presence differences.
A missing or mismatched staged result for `outcome_ready` is presentation corruption and must fault snapshot capture.
## Runtime event production changes
Primary file: `packages/agent/src/harness/runtime/drive/tools.ts`
Related helpers: `packages/agent/src/harness/execution/tools.ts` and `packages/agent/src/harness/runtime/drive/tool-placement.ts`.
### Internal outcome shape
Current:
```ts
type ToolOutcome = { message: ToolResultMessage<unknown>; terminate: boolean };
```
Extend/refactor it so post-commit event production has the complete canonical final result and `isError`, without lossy reconstruction. It must retain enough data for:
- staged `ToolResultMessage`;
- `tool_end.result`;
- `tool_end.isError`;
- durable/effective `terminate` after cancellation normalization.
Synthetic helpers currently return `ToolResultMessage` directly. Refactor carefully so synthetic outcomes also carry the canonical result data. Do not invent `details` in the transcript: existing unknown/invalid synthetic results deliberately omit message details.
### Commit-bound `tool_start`
For fresh execution, `publishToolIntent()` attaches `tool_start` to the commit that persists effective arguments and changes the call to `effect_pending`. The public operation awaits delivery before admitting `executeToolCall()`, preserving `tool_start → tool_update*` without requiring each update callback to await delivery.
For a fresh synthetic call that never writes effect intent, `publishToolOutcome()` attaches `tool_start` before `tool_end` in the outcome-staging commit's event batch. It reports the source block arguments.
For safe recovery, the checkpoint-clear commit emits recovery-tagged `tool_start` using persisted effective arguments. Do not emit a fresh start for an already-restored unsafe `effect_pending` call; its initial snapshot is the baseline.
### Post-commit `tool_end`
Remove the current pre-staging `tool_end` emission from `performToolInvocation()`.
`publishToolOutcome()` attaches `tool_end` to the same staging command's `events` callback. The event carries:
- `runId`, `turnId`, `toolCallId`, `toolName`;
- canonical final `result`;
- `isError`;
- cancellation-normalized durable `terminate`;
- `recovery: true` where applicable.
Arguments belong to `tool_start` and are not repeated on `tool_end`. Event data describes the state actually committed, especially cancellation forcing `terminate: false`.
Because `Lane.command()` awaits retained event delivery and `runParallel()` schedules materialization from the outcome-completion promise, the required order is:
```text
staging commit
→ tool_end delivery
→ source-ready message lifecycle
→ placement commit
→ entry_added
```
### Call sites to audit
Every `publishToolOutcome()` call must supply the source tool call and recovery context correctly:
- immediate outcome in `startToolInvocation()`;
- cancellation after intent but before execution;
- normal `performToolInvocation()` completion;
- safe replay completion;
- unsafe recovery interruption;
- sequential cancellation of `planned`;
- sequential cancellation of `effect_pending`.
Also audit the synchronous `AbortRequested` path inside `performToolInvocation()`: a call with durable intent must still have coherent start/end presentation even if effect admission fails immediately.
## Mini and other presentation consumers
### Mini
File: `packages/coding-agent/src/experimental/mini/tui/view.ts`, `MiniTui.apply()`.
For each `runningTools` row:
```text
status running:
markExecutionStarted()
if result exists: updateResult({...result, isError:false}, true)
status settled:
do not call markExecutionStarted()
updateResult({...result, isError}, false)
```
The final result remains visible while awaiting placement. After `entry_added`, transcript synchronization supplies the immutable `ToolResultMessage` and the row is no longer in `runningTools`.
### Other shared consumer
Apply equivalent handling in:
- `packages/coding-agent/src/experimental/client-tui-chat.ts`
Read `packages/coding-agent/src/modes/interactive/components/tool-execution.ts` before editing to confirm `updateResult(result, isPartial)` semantics.
## Tests
### Reducer tests
File: `packages/agent/test/harness/runtime/reducer.test.ts`
Add a parallel batch event-fold test with calls 0, 1, 2:
1. Establish all three as running.
2. Call 2 settles before call 0.
3. Call 0 settles and is placed while call 1 still runs.
4. Call 1 settles.
5. Placement flushes calls 1 and 2 in source order.
6. After every settlement and placement event, assert each presentation-active call appears in exactly one of:
- `operation.runningTools`; or
- transcript `toolResult` entries.
7. Assert call 2 remains present with `status:"settled"` and its final result while blocked by earlier calls.
8. Assert each `entry_added` removes only its matching active row.
Add focused coverage for:
- `tool_update` from a stale/wrong `runId` does not mutate the current operation;
- `tool_start` upserts rather than duplicates a row captured from durable intent;
- `tool_end` settles the existing batch-local row.
### Capture/watch tests
File: `packages/agent/test/harness/runtime/watch.test.ts`
Construct a durable tools state containing:
- `planned` (omitted);
- `effect_pending` with checkpoint (`status:"running"`, checkpoint exposed as `result`);
- `effect_pending` without checkpoint;
- real `outcome_ready` with persisted effective args;
- synthetic `outcome_ready` without `operationToolArgs`, falling back to source block arguments;
- completed call represented only by a transcript entry where practical.
Assert staged settled content, `isError`, arguments, and absence of duplicates. Add missing/mismatched `pendingEntry` corruption assertions.
### Runtime tool tests
File: `packages/agent/test/harness/runtime/drive-tools.test.ts`
Update/add ordering assertions proving:
- real: `intent commit < tool_start < tool_update* < staging commit < tool_end < entry_added`;
- immediate synthetic: `staging commit < tool_start < tool_end < entry_added`, no tool effect and no `after_tool`;
- planned cancellation gets coherent start/end;
- unsafe recovery uses initial snapshot plus recovery-tagged end without replaying effects;
- B can emit post-commit end and remain settled while A blocks placement;
- source-order placement remains unchanged;
- a crash after staging cannot replay the call.
The baseline normative tests/documentation required `tool_end` before staging; the implementation reverses those expectations deliberately.
### Type/event catalog tests
Audit:
- `packages/agent/test/harness/types.test.ts`
- `packages/agent/src/harness/telemetry.ts`
No new event name is added. `tool_end` omits arguments and its semantics change.
### Mini regression
After unit tests, run the real mini abort smoke test used previously:
- execute exactly `sleep 20`;
- send Escape after about two seconds;
- assert `Command aborted` and elapsed time appear;
- assert raw ANSI contains `toolErrorBg` (`48;2;60;40;40`);
- verify the durable session contains an `isError:true` tool result and operation status `aborted`.
Also exercise a parallel batch where a later tool finishes first and verify its final result remains visible until in-order placement.
## Documentation changes
Read both documents completely before editing:
- `packages/agent/docs/harness.md` (normative)
- `packages/agent/docs/tool-durability.md`
The implementation updates baseline statements that required:
- `tool_end` before staging;
- `tool_start`/`tool_end` only for real effects;
- synthetic outcomes emitting no lifecycle;
- `outcome_ready` being omitted from `runningTools`;
- snapshot field `partialResult`.
Important known locations from the baseline:
- `harness.md` §3.8 around lines 784–796;
- `harness.md` §5.4 `LaneSnapshot` around lines 1101–1134;
- `harness.md` §5.5 events around lines 1140–1158;
- `harness.md` tool phases around lines 1214–1224;
- `harness.md` conformance requirements around lines 1386–1400;
- `tool-durability.md` finalization around lines 255–280;
- `tool-durability.md` snapshots/events around lines 568–584;
- `tool-durability.md` test requirements around lines 671–679.
The revised docs must state:
- `tool_end` is post-staging durability evidence for a final result;
- fresh synthetic outcomes receive start/end lifecycle;
- recovery events are not historically replayed;
- `outcome_ready` remains projected as settled until placement;
- `entry_added` moves settled presentation into transcript;
- the unified snapshot `result` is provisional when running and final when settled.
Do not modify the separate `response.ts`/`tool-placement.ts` recovery `turn_end` discrepancy unless separately requested.
## Files to reread completely after compaction
Core/specification:
1. `packages/agent/docs/harness.md`
2. `packages/agent/docs/tool-durability.md`
3. `packages/agent/src/harness/agent-harness.ts`
4. `packages/agent/src/harness/runtime/reducer.ts`
5. `packages/agent/src/harness/runtime/lane.ts`
6. `packages/agent/src/harness/runtime/drive/tools.ts`
7. `packages/agent/src/harness/runtime/drive/tool-placement.ts`
8. `packages/agent/src/harness/execution/tools.ts`
9. `packages/agent/src/harness/runtime/types.ts`
10. `packages/agent/src/harness/session/types.ts`
11. `packages/agent/src/harness/events.ts`
12. `packages/agent/src/harness/telemetry.ts`
Tests:
13. `packages/agent/test/harness/runtime/reducer.test.ts`
14. `packages/agent/test/harness/runtime/watch.test.ts`
15. `packages/agent/test/harness/runtime/drive-tools.test.ts`
16. `packages/agent/test/harness/types.test.ts`
17. Relevant test helpers imported by those files.
Mini/presentation:
18. `packages/coding-agent/src/experimental/mini/tui/session.ts`
19. `packages/coding-agent/src/experimental/mini/worker/lane-service.ts`
20. `packages/coding-agent/src/experimental/mini/shared/protocol.ts`
21. `packages/coding-agent/src/experimental/mini/tui/view.ts`
22. `packages/coding-agent/src/experimental/client-tui-chat.ts`
23. `packages/coding-agent/src/modes/interactive/components/tool-execution.ts`
Before editing, run `git status --short` and inspect current diffs because other Pi sessions may share the worktree.
## Validation commands
From repository root, after changes:
```bash
cd packages/agent
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/harness/runtime/reducer.test.ts
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/harness/runtime/watch.test.ts
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/harness/runtime/drive-tools.test.ts
node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/harness/types.test.ts
cd "$(git rev-parse --show-toplevel)"
npm run check
```
Do not run `npm test`, the full Vitest suite, or `npm run build` unless requested.
If a delegated review is used, repository policy requires:
```text
--provider anthropic --model claude-fable-5
```
Keep extensions enabled.
## Non-goals
- No generic structural-delta transport for mini.
- No optimization to avoid the final result crossing the wire once in `tool_end` and again in `entry_added`.
- No change to source-prefix placement semantics.
- No clearing on `turn_end`.
- No change to tool effect replay/durability rules.
- No change to `after_tool`: it still runs only for actual fresh/safely replayed effects under its existing cancellation contract.
- No work on the separate recovery `turn_end` discrepancy between `response.ts` and `tool-placement.ts`.
- No backward-compatibility layer unless the user explicitly asks for one.
- Do not commit unless the user asks.

50
third_party/pi/packages/agent/examples/mcp-codemode/main.ts vendored Normal file
View File

@@ -0,0 +1,50 @@
/**
* An Agent with the tools of an MCP server plus a codemode tool that calls them from JavaScript.
*
* ANTHROPIC_API_KEY=... node packages/agent/examples/mcp-codemode/main.ts \
* npx -y @modelcontextprotocol/server-filesystem .
*
* The arguments are the command that starts a stdio MCP server.
*/
import { Agent } from "@earendil-works/pi-agent-core";
import { createModels } from "@earendil-works/pi-ai";
import { anthropicProvider } from "@earendil-works/pi-ai/providers/anthropic";
import { McpClient, StdioTransport } from "@earendil-works/pi-mcp";
import { createCodemodeTool, createMcpTools, createNestedToolRunner } from "./tools.ts";
const [command, ...args] = process.argv.slice(2);
if (!command) {
console.error("Usage: main.ts <mcp server command> [args...]");
process.exit(1);
}
const models = createModels();
models.setProvider(anthropicProvider());
const model = models.getModel("anthropic", "claude-sonnet-4-6");
if (!model) throw new Error("Model not found");
const client = new McpClient({ name: "pi-agent-example", version: "1.0.0" });
await client.connect(new StdioTransport({ command, args }));
try {
const agent = new Agent({
initialState: { systemPrompt: "You are a helpful assistant.", model },
streamFn: models.streamSimple.bind(models),
});
const mcpTools = await createMcpTools(client, "mcp");
// The model can call the MCP tools directly or batch them in one codemode script.
agent.state.tools = [...mcpTools, createCodemodeTool(mcpTools, createNestedToolRunner(agent))];
agent.subscribe((event) => {
if (event.type === "message_update" && event.assistantMessageEvent.type === "text_delta") {
process.stdout.write(event.assistantMessageEvent.delta);
} else if (event.type === "tool_execution_start") {
console.log(`\n[${event.toolName}]`);
}
});
await agent.prompt("Use codemode to list what the MCP tools can do, then summarize it in two sentences.");
console.log();
} finally {
await client.close();
}

172
third_party/pi/packages/agent/examples/mcp-codemode/tools.ts vendored Normal file
View File

@@ -0,0 +1,172 @@
/**
* Adapters that give a pi-agent-core `Agent` MCP tools and a codemode tool, using
* `@earendil-works/pi-mcp` and `@earendil-works/pi-codemode`. See main.ts for how they fit together.
*/
import {
type Agent,
type AgentTool,
type AgentToolCallOutcome,
type AgentToolResult,
runToolCall,
} from "@earendil-works/pi-agent-core";
import type { ImageContent, JsonObject, JsonValue, TextContent } from "@earendil-works/pi-ai";
import {
type CodemodeJsonSchema,
CodemodeSandbox,
type CodemodeTool,
renderDeclarations,
} from "@earendil-works/pi-codemode";
import { type McpClient, toLlmContent } from "@earendil-works/pi-mcp";
import { Type } from "typebox";
/** Provider tool names are limited to 64 characters of `[A-Za-z0-9_-]`. */
function toToolName(prefix: string, name: string): string {
return `${prefix}_${name}`.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 64);
}
function textOf(result: AgentToolResult<unknown>): string {
return result.content
.filter((block): block is TextContent => block.type === "text")
.map((block) => block.text)
.join("\n");
}
/**
* One `AgentTool` per tool of a connected MCP server, named `<prefix>_<tool>`. Tools with an MCP
* output schema declare it as `outputSchema` and return `structuredContent`, which codemode scripts
* receive instead of the text.
*/
export async function createMcpTools(client: McpClient, prefix: string): Promise<AgentTool[]> {
const tools = await client.listTools();
return tools.map(
(tool): AgentTool => ({
name: toToolName(prefix, tool.name),
label: tool.title ?? tool.name,
description: tool.description ?? tool.title ?? `MCP tool ${tool.name}`,
// Providers require an object schema, and some reject one without `properties`.
parameters: Type.Unsafe({
...tool.inputSchema,
type: "object",
properties: tool.inputSchema.properties ?? {},
}),
outputSchema: tool.outputSchema ? Type.Unsafe<Record<string, unknown>>(tool.outputSchema) : undefined,
execute: async (_toolCallId, params, signal, onUpdate) => {
const result = await client.callTool(tool.name, params as Record<string, unknown>, {
signal,
onProgress: (progress) => {
const text = progress.message ?? `Progress ${progress.progress}`;
onUpdate?.({ content: [{ type: "text", text }], details: undefined });
},
});
const content = toLlmContent(result);
if (result.isError && content.length === 0) {
content.push({ type: "text", text: `MCP tool ${tool.name} failed` });
}
return {
content,
details: undefined,
structuredContent: result.structuredContent as JsonValue | undefined,
// MCP reports tool failures in the result instead of as a protocol error.
isError: result.isError === true,
};
},
}),
);
}
/**
* Run a tool call on behalf of another tool through the agent's pipeline: argument validation and
* the agent's `beforeToolCall`/`afterToolCall` hooks apply as for calls the model makes. Calling
* `tool.execute()` directly also works when the agent has no hooks.
*/
export function createNestedToolRunner(
agent: Agent,
): (name: string, args: unknown, signal: AbortSignal) => Promise<AgentToolCallOutcome> {
let nextId = 1;
return (name, args, signal) => {
const assistantMessage = agent.state.messages.findLast((message) => message.role === "assistant");
if (assistantMessage?.role !== "assistant") throw new Error("Nested tool calls need an assistant message");
const tools = agent.state.tools;
return runToolCall(
{ type: "toolCall", id: `nested-${nextId++}`, name, arguments: (args ?? {}) as JsonObject },
{
tools,
assistantMessage,
context: { messages: agent.state.messages, tools },
beforeToolCall: agent.beforeToolCall,
afterToolCall: agent.afterToolCall,
signal,
},
);
};
}
export const CODEMODE_TOOL_NAME = "codemode";
const codemodeParameters = Type.Object({
code: Type.String({ description: "JavaScript source. Top-level await and return work." }),
});
/**
* A tool that runs model-written JavaScript in the codemode sandbox, where `tools.<name>(args)`
* calls `tools`. Nested results stay out of the model context; only the script's output and
* return value come back.
*/
export function createCodemodeTool(
tools: readonly AgentTool[],
runTool: (name: string, args: unknown, signal: AbortSignal) => Promise<AgentToolCallOutcome>,
): AgentTool<typeof codemodeParameters> {
const callable = tools.filter((tool) => tool.name !== CODEMODE_TOOL_NAME);
const sandboxTools: CodemodeTool[] = callable.map((tool) => ({
name: tool.name,
description: tool.description,
// TypeBox schemas are JSON Schema.
inputSchema: tool.parameters as CodemodeJsonSchema,
// What the script receives: the structured result if the tool declares one, else its text.
outputSchema: (tool.outputSchema as CodemodeJsonSchema | undefined) ?? { type: "string" },
execute: async (args, { signal }) => {
const outcome = await runTool(tool.name, args, signal);
if (tool.outputSchema && outcome.result.structuredContent !== undefined) {
return outcome.result.structuredContent;
}
const text = textOf(outcome.result);
if (outcome.isError) throw new Error(text || `Tool ${tool.name} failed`);
return text;
},
}));
return {
name: CODEMODE_TOOL_NAME,
label: "Codemode",
description: [
"Run JavaScript that calls other tools, to chain calls, run them in parallel, or filter large results.",
"The code is the body of an async function. There is no Node, file system, network, or timers.",
"Call tools as `await tools.<name>(args)`. Output with `text(value)` or `console.log()`, or `return` a value.",
"Only the output and the return value reach you, not the results of the tool calls.",
"",
"```ts",
renderDeclarations({ tools: sandboxTools }),
"```",
].join("\n"),
parameters: codemodeParameters,
execute: async (_toolCallId, params, signal) => {
const sandbox = new CodemodeSandbox({ tools: sandboxTools, timeoutMs: 120_000 });
try {
const result = await sandbox.execute(params.code, { signal });
// Items from text(), console.*, and image(), in order. They have pi-ai's content shapes.
const content: (TextContent | ImageContent)[] = [...result.output];
if (result.ok && result.value !== undefined) {
const value = typeof result.value === "string" ? result.value : JSON.stringify(result.value);
content.push({ type: "text", text: value });
}
if (!result.ok) {
content.push({ type: "text", text: `Script failed: ${result.error.stack ?? result.error.message}` });
}
return { content, details: { calls: result.calls }, isError: !result.ok };
} finally {
await sandbox.close();
}
},
};
}

49
third_party/pi/packages/agent/package.json vendored
View File

@@ -1,6 +1,6 @@
{
"name": "@earendil-works/pi-agent-core",
"version": "0.85.1",
"version": "1.3.0",
"description": "General-purpose agent with transport abstraction, state management, and attachment support",
"type": "module",
"main": "./dist/index.js",
@@ -10,30 +10,6 @@
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
},
"./node": {
"types": "./dist/node.d.ts",
"import": "./dist/node.js"
},
"./harness/context": {
"types": "./dist/harness/context.d.ts",
"import": "./dist/harness/context.js"
},
"./harness/env/nodejs": {
"types": "./dist/harness/env/nodejs.d.ts",
"import": "./dist/harness/env/nodejs.js"
},
"./harness/runtime/reducer": {
"types": "./dist/harness/runtime/reducer.d.ts",
"import": "./dist/harness/runtime/reducer.js"
},
"./harness/session": {
"types": "./dist/harness/session/index.d.ts",
"import": "./dist/harness/session/index.js"
},
"./harness/session/testing": {
"types": "./dist/harness/session/testing/index.d.ts",
"import": "./dist/harness/session/testing/index.js"
},
"./package.json": "./package.json"
},
"files": [
@@ -42,26 +18,13 @@
],
"scripts": {
"clean": "shx rm -rf dist",
"generate-telemetry-docs": "node scripts/generate-telemetry-docs.ts",
"check:telemetry-docs": "node scripts/generate-telemetry-docs.ts --check",
"build": "tsgo -p tsconfig.build.json",
"bench:session:timing": "vitest bench --config vitest.benchmark.config.ts",
"bench:session:memory": "tsx --tsconfig benchmark/tsconfig.json benchmark/session/loaded-footprint.ts",
"bench:session:allocations": "tsx --tsconfig benchmark/tsconfig.json benchmark/session/allocation-profile.ts",
"build": "tsc -p tsconfig.build.json",
"test": "vitest --run",
"test:harness": "vitest --run --config vitest.harness.config.ts",
"test:session:conformance": "vitest --run --config vitest.harness.config.ts test/harness/memory-conformance.test.ts",
"coverage:harness": "vitest --run --config vitest.harness.config.ts --coverage",
"prepublishOnly": "npm run build"
},
"dependencies": {
"@earendil-works/chord": "^0.85.1",
"@earendil-works/pi-ai": "^0.85.1",
"@earendil-works/pi-telemetry": "^0.85.1",
"diff": "8.0.4",
"ignore": "7.0.5",
"typebox": "1.3.7",
"yaml": "2.9.0"
"@earendil-works/pi-ai": "1.3.0",
"typebox": "1.3.27"
},
"keywords": [
"ai",
@@ -82,8 +45,6 @@
},
"devDependencies": {
"@types/node": "22.19.19",
"@vitest/coverage-v8": "4.1.9",
"typescript": "5.9.3",
"vitest": "4.1.9"
"vitest": "4.1.11"
}
}

117
third_party/pi/packages/agent/scripts/generate-telemetry-docs.ts vendored
View File

@@ -1,117 +0,0 @@
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { pathToFileURL } from "node:url";
import type { TelemetryAttributeDefinition, TelemetrySchemaDefinition } from "@earendil-works/pi-telemetry";
import { AI_TELEMETRY_SCHEMA, HARNESS_TELEMETRY_SCHEMA } from "../src/harness/telemetry.ts";
function escapeCell(value: string): string {
return value.replaceAll("|", "\\|").replaceAll("\n", " ");
}
function allowedValues(definition: TelemetryAttributeDefinition): string {
if ("values" in definition && definition.values) return definition.values.map(String).join(", ");
if ("elementValues" in definition && definition.elementValues) {
return `elements: ${definition.elementValues.map(String).join(", ")}`;
}
return "";
}
function attributeNotes(definition: TelemetryAttributeDefinition): string {
return [definition.cardinality ? `${definition.cardinality} cardinality` : "", definition.sensitive ? "sensitive" : ""]
.filter(Boolean)
.join(", ");
}
function parentDescription(parent: TelemetrySchemaDefinition["spans"][string]["parents"]): string {
switch (parent.kind) {
case "any":
return "root or any caller span";
case "root_or_external":
return "root or caller-owned external span";
case "spans":
return parent.spans.map((span) => `\`${span}\``).join(", ");
}
}
function renderSchema(schema: TelemetrySchemaDefinition, title: string): string[] {
const lines = [`## ${title}`, "", `Schema version: ${schema.version}`, ""];
for (const [spanName, span] of Object.entries(schema.spans)) {
lines.push(`### \`${spanName}\``, "", span.description, "");
lines.push(`- Parents: ${parentDescription(span.parents)}`);
lines.push(`- Default status: \`${span.status.default}\``);
lines.push(`- Error when: ${span.status.errorWhen}`, "");
lines.push("#### Start attributes", "");
lines.push("| Name | Type | Required | Values | Notes | Description |");
lines.push("|---|---|---:|---|---|---|");
for (const [name, definition] of Object.entries(span.startAttributes)) {
lines.push(
`| \`${name}\` | \`${definition.type}\` | ${definition.required ? "yes" : "no"} | ${escapeCell(allowedValues(definition))} | ${escapeCell(attributeNotes(definition))} | ${escapeCell(definition.description)} |`,
);
}
if (Object.keys(span.startAttributes).length === 0) lines.push("| _none_ | | | | | |");
lines.push("", "#### End attributes", "");
lines.push("All end attributes are optional completion enrichment.", "");
lines.push("| Name | Type | Values | Notes | Description |");
lines.push("|---|---|---|---|---|");
for (const [name, definition] of Object.entries(span.endAttributes)) {
lines.push(
`| \`${name}\` | \`${definition.type}\` | ${escapeCell(allowedValues(definition))} | ${escapeCell(attributeNotes(definition))} | ${escapeCell(definition.description)} |`,
);
}
if (Object.keys(span.endAttributes).length === 0) lines.push("| _none_ | | | | |");
lines.push("", "#### Events", "");
const events = Object.entries(span.events ?? {});
if (events.length === 0) {
lines.push("No declared span events.", "");
continue;
}
for (const [eventName, event] of events) {
lines.push(`##### \`${eventName}\``, "", event.description, "");
lines.push("| Name | Type | Required | Values | Notes | Description |");
lines.push("|---|---|---:|---|---|---|");
for (const [name, definition] of Object.entries(event.attributes)) {
lines.push(
`| \`${name}\` | \`${definition.type}\` | ${definition.required ? "yes" : "no"} | ${escapeCell(allowedValues(definition))} | ${escapeCell(attributeNotes(definition))} | ${escapeCell(definition.description)} |`,
);
}
if (Object.keys(event.attributes).length === 0) lines.push("| _none_ | | | | | |");
lines.push("");
}
}
return lines;
}
export function renderAgentTelemetrySchemaMarkdown(): string {
const lines = [
"# Pi Agent Telemetry Schemas",
"",
"<!-- Generated by generate-telemetry-docs.ts. Do not edit manually. -->",
"",
...renderSchema(AI_TELEMETRY_SCHEMA, "AI request schema"),
...renderSchema(HARNESS_TELEMETRY_SCHEMA, "Harness schema"),
];
return `${lines.join("\n").trimEnd()}\n`;
}
export function generateTelemetryDocs(outputPath: string, check: boolean): void {
const expected = renderAgentTelemetrySchemaMarkdown();
if (check) {
let actual = "";
try {
actual = readFileSync(outputPath, "utf8");
} catch {
throw new Error(`${outputPath} is missing; run the telemetry documentation generator`);
}
if (actual !== expected) throw new Error(`${outputPath} is stale; run the telemetry documentation generator`);
return;
}
mkdirSync(dirname(outputPath), { recursive: true });
writeFileSync(outputPath, expected);
}
if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) {
generateTelemetryDocs(
resolve(import.meta.dirname, "../docs/telemetry-schema.md"),
process.argv.includes("--check"),
);
}

246
third_party/pi/packages/agent/src/agent-loop.ts vendored
View File

@@ -5,9 +5,14 @@
import {
type AssistantMessage,
type Context,
EventStream,
getCurrentTools,
getToolStateChanges,
normalizeContext,
type SystemMessage,
type ToolResultMessage,
type ToolStateChanges,
toToolDeclaration,
validateToolArguments,
} from "@earendil-works/pi-ai";
import { getDefaultStreamFn } from "./stream-fn.ts";
@@ -18,6 +23,7 @@ import type {
AgentMessage,
AgentTool,
AgentToolCall,
AgentToolCallOutcome,
AgentToolResult,
PrepareNextTurnContext,
StreamFn,
@@ -101,17 +107,18 @@ export async function runAgentLoop(
signal: AbortSignal | undefined,
streamFn: StreamFn,
): Promise<AgentMessage[]> {
const newMessages: AgentMessage[] = [...prompts];
const initialMessages = declareToolChanges(context, prompts);
const newMessages: AgentMessage[] = [...initialMessages];
const currentContext: AgentContext = {
...context,
messages: [...context.messages, ...prompts],
messages: [...context.messages, ...initialMessages],
};
await emit({ type: "agent_start" });
await emit({ type: "turn_start" });
for (const prompt of prompts) {
await emit({ type: "message_start", message: prompt });
await emit({ type: "message_end", message: prompt });
for (const message of initialMessages) {
await emit({ type: "message_start", message });
await emit({ type: "message_end", message });
}
await runLoop(currentContext, newMessages, config, signal, emit, streamFn ?? getDefaultStreamFn());
@@ -164,6 +171,7 @@ async function runLoop(
let currentContext = initialContext;
let config = initialConfig;
let lastCompletedTurn: PrepareNextTurnContext | undefined;
let explicitContinuation = false;
// Check for steering messages at start (user may have typed while waiting)
let pendingMessages: AgentMessage[] = (await config.getSteeringMessages?.()) || [];
@@ -173,10 +181,12 @@ async function runLoop(
// Inner loop: process tool calls and steering messages
while (hasMoreToolCalls || pendingMessages.length > 0) {
let preparedMessages: AgentMessage[] = [];
if (lastCompletedTurn) {
const nextTurnSnapshot = await config.prepareNextTurn?.(lastCompletedTurn);
if (nextTurnSnapshot) {
currentContext = nextTurnSnapshot.context ?? currentContext;
preparedMessages = nextTurnSnapshot.messages ?? [];
config = {
...config,
model: nextTurnSnapshot.model ?? config.model,
@@ -197,15 +207,35 @@ async function runLoop(
await emit({ type: "turn_start" });
}
// Process pending messages (inject before next assistant response)
if (pendingMessages.length > 0) {
for (const message of pendingMessages) {
await emit({ type: "message_start", message });
await emit({ type: "message_end", message });
currentContext.messages.push(message);
newMessages.push(message);
}
pendingMessages = [];
// Process prepared and queued messages before the next assistant response.
for (const message of declareToolChanges(currentContext, [...preparedMessages, ...pendingMessages])) {
await emit({ type: "message_start", message });
await emit({ type: "message_end", message });
currentContext.messages.push(message);
newMessages.push(message);
}
pendingMessages = [];
const requestUpdate = await config.prepareRequest?.(
{
context: currentContext,
model: config.model,
thinkingLevel: config.reasoning ?? "off",
},
signal,
);
if (requestUpdate) {
currentContext = requestUpdate.context ?? currentContext;
config = {
...config,
model: requestUpdate.model ?? config.model,
reasoning:
requestUpdate.thinkingLevel === undefined
? config.reasoning
: requestUpdate.thinkingLevel === "off"
? undefined
: requestUpdate.thinkingLevel,
};
}
// Stream assistant response
@@ -213,6 +243,13 @@ async function runLoop(
newMessages.push(message);
if (message.stopReason === "error" || message.stopReason === "aborted") {
lastCompletedTurn = {
message,
toolResults: [],
context: currentContext,
newMessages,
};
await config.finishTurn?.(lastCompletedTurn, signal);
await emit({ type: "turn_end", message, toolResults: [] });
await emit({ type: "agent_end", messages: newMessages });
return;
@@ -240,31 +277,42 @@ async function runLoop(
}
}
await emit({ type: "turn_end", message, toolResults });
lastCompletedTurn = {
message,
toolResults,
context: currentContext,
newMessages,
};
const decision = await config.finishTurn?.(lastCompletedTurn, signal);
await emit({ type: "turn_end", message, toolResults });
if (await config.shouldStopAfterTurn?.(lastCompletedTurn)) {
if (decision?.action === "end") {
await emit({ type: "agent_end", messages: newMessages });
return;
}
explicitContinuation = decision?.action === "continue";
pendingMessages = (await config.getSteeringMessages?.()) || [];
if (hasMoreToolCalls || pendingMessages.length > 0) {
explicitContinuation = false;
}
}
// Agent would stop here. Check for follow-up messages.
const followUpMessages = (await config.getFollowUpMessages?.()) || [];
if (followUpMessages.length > 0) {
// Set as pending so inner loop processes them
explicitContinuation = false;
pendingMessages = followUpMessages;
continue;
}
// No natural request was selected, so fulfill the continuation decision with one context-only turn.
if (explicitContinuation) {
explicitContinuation = false;
continue;
}
// No more messages, exit
break;
}
@@ -272,6 +320,60 @@ async function runLoop(
await emit({ type: "agent_end", messages: newMessages });
}
/**
* Declare tool loadout changes to the model.
*
* `context.tools` is what the runtime can execute; the transcript's system messages declare
* what the model may call. Before each request the difference becomes `toolsAdded` and
* `toolsRemoved` on a system message. When a pending system message exists, its tool fields
* are treated as intent and replaced with the delta between the committed transcript and
* the executable set, so replay always yields exactly `context.tools`. Otherwise a new
* system message is inserted before the first non-system pending message.
*/
function declareToolChanges(context: AgentContext, pendingMessages: AgentMessage[]): AgentMessage[] {
let systemIndex = -1;
for (let i = pendingMessages.length - 1; i >= 0; i--) {
if (pendingMessages[i].role === "system") {
systemIndex = i;
break;
}
}
const pending = pendingMessages[systemIndex] as SystemMessage | undefined;
const baseline = pending
? pendingMessages.map((message, index) =>
index === systemIndex ? withToolChanges(pending, NO_CHANGES) : message,
)
: pendingMessages;
const changes = getToolStateChanges(
getCurrentTools([...context.messages, ...baseline]),
(context.tools ?? []).map(toToolDeclaration),
);
const unchanged = changes.toolsAdded.length === 0 && changes.toolsRemoved.length === 0;
if (pending) {
// Keep the caller's message object when it already declares no tool changes.
if (unchanged && !pending.toolsAdded?.length && !pending.toolsRemoved?.length) return pendingMessages;
return baseline.map((message, index) => (index === systemIndex ? withToolChanges(pending, changes) : message));
}
if (unchanged) return pendingMessages;
const update = withToolChanges({ role: "system", content: "", timestamp: Date.now() }, changes);
const insertIndex = pendingMessages.findIndex((message) => message.role !== "system");
const index = insertIndex === -1 ? pendingMessages.length : insertIndex;
return [...pendingMessages.slice(0, index), update, ...pendingMessages.slice(index)];
}
const NO_CHANGES: ToolStateChanges = { toolsAdded: [], toolsRemoved: [] };
/** Copy a system message with its tool fields replaced by `changes`; empty lists omit the field. */
function withToolChanges(message: SystemMessage, { toolsAdded, toolsRemoved }: ToolStateChanges): SystemMessage {
const { toolsAdded: _added, toolsRemoved: _removed, ...rest } = message;
return {
...rest,
...(toolsAdded.length > 0 ? { toolsAdded } : {}),
...(toolsRemoved.length > 0 ? { toolsRemoved } : {}),
};
}
/**
* Stream an assistant response from the LLM.
* This is where AgentMessage[] gets transformed to Message[] for the LLM.
@@ -292,12 +394,7 @@ async function streamAssistantResponse(
// Convert to LLM-compatible messages (AgentMessage[] → Message[])
const llmMessages = await config.convertToLlm(messages);
// Build LLM context
const llmContext: Context = {
systemPrompt: context.systemPrompt,
messages: llmMessages,
tools: context.tools,
};
const llmContext = normalizeContext({ messages: llmMessages });
// Resolve API key (important for expiring tokens)
const resolvedApiKey =
@@ -308,6 +405,8 @@ async function streamAssistantResponse(
apiKey: resolvedApiKey,
signal,
});
// Record the requested level, whichever stream function answered.
const result = async () => Object.assign(await response.result(), { thinkingLevel: config.reasoning ?? "off" });
let partialMessage: AssistantMessage | null = null;
let addedPartial = false;
@@ -343,7 +442,7 @@ async function streamAssistantResponse(
case "done":
case "error": {
const finalMessage = await response.result();
const finalMessage = await result();
if (addedPartial) {
context.messages[context.messages.length - 1] = finalMessage;
} else {
@@ -358,7 +457,7 @@ async function streamAssistantResponse(
}
}
const finalMessage = await response.result();
const finalMessage = await result();
if (addedPartial) {
context.messages[context.messages.length - 1] = finalMessage;
} else {
@@ -456,7 +555,7 @@ async function executeToolCallsSequential(
isError: preparation.isError,
};
} else {
const executed = await executePreparedToolCall(preparation, signal, emit);
const executed = await executePreparedToolCall(preparation, signal, emitToolExecutionUpdate(toolCall, emit));
finalized = await finalizeExecutedToolCall(
currentContext,
assistantMessage,
@@ -527,7 +626,7 @@ async function executeToolCallsParallel(
await emitToolExecutionEnd(finalized, emit);
return finalized;
}
const executed = await executePreparedToolCall(preparation, signal, emit);
const executed = await executePreparedToolCall(preparation, signal, emitToolExecutionUpdate(toolCall, emit));
const finalized = await finalizeExecutedToolCall(
currentContext,
assistantMessage,
@@ -576,13 +675,15 @@ type ImmediateToolCallOutcome = {
type ExecutedToolCallOutcome = {
result: AgentToolResult<any>;
isError: boolean;
durationMs: number;
};
type FinalizedToolCallOutcome = {
toolCall: AgentToolCall;
result: AgentToolResult<any>;
isError: boolean;
};
type FinalizedToolCallOutcome = AgentToolCallOutcome;
/** The `beforeToolCall` and `afterToolCall` hooks of {@link AgentLoopConfig}. */
export type ToolCallHooks = Pick<AgentLoopConfig, "beforeToolCall" | "afterToolCall">;
type ToolUpdateSink = (partialResult: AgentToolResult<any>) => Promise<void> | void;
type FinalizedToolCallEntry = FinalizedToolCallOutcome | (() => Promise<FinalizedToolCallOutcome>);
@@ -608,10 +709,11 @@ async function prepareToolCall(
currentContext: AgentContext,
assistantMessage: AssistantMessage,
toolCall: AgentToolCall,
config: AgentLoopConfig,
config: ToolCallHooks,
signal: AbortSignal | undefined,
tools: readonly AgentTool<any>[] = currentContext.tools ?? [],
): Promise<PreparedToolCall | ImmediateToolCallOutcome> {
const tool = currentContext.tools?.find((t) => t.name === toolCall.name);
const tool = tools.find((t) => t.name === toolCall.name);
if (!tool) {
return {
kind: "immediate",
@@ -674,13 +776,57 @@ async function prepareToolCall(
}
}
function emitToolExecutionUpdate(toolCall: AgentToolCall, emit: AgentEventSink): ToolUpdateSink {
return (partialResult) =>
emit({
type: "tool_execution_update",
toolCallId: toolCall.id,
toolName: toolCall.name,
args: toolCall.arguments,
partialResult,
});
}
/** Options for {@link runToolCall}. */
export interface RunToolCallOptions extends ToolCallHooks {
/** Tools the call resolves against. */
tools: readonly AgentTool<any>[];
/** Passed to the hooks as the message that issued the call. */
assistantMessage: AssistantMessage;
/** Passed to the hooks as the current agent context. */
context: AgentContext;
signal?: AbortSignal;
onUpdate?: ToolUpdateSink;
}
/**
* Run one tool call through the same steps as a model-issued call: argument preparation, schema
* validation, `beforeToolCall`, execution, and `afterToolCall`. Emits no events and adds no
* messages. Tools that call other tools use this so the hooks (for example permission checks)
* apply to those calls too.
*
* Never rejects for tool failures: unknown tools, validation errors, blocked calls, and thrown
* errors come back as `isError: true`.
*/
export async function runToolCall(toolCall: AgentToolCall, options: RunToolCallOptions): Promise<AgentToolCallOutcome> {
const { assistantMessage, context, signal } = options;
const preparation = await prepareToolCall(context, assistantMessage, toolCall, options, signal, options.tools);
if (preparation.kind === "immediate") {
return { toolCall, result: preparation.result, isError: preparation.isError };
}
const executed = await executePreparedToolCall(preparation, signal, options.onUpdate ?? (() => {}));
return finalizeExecutedToolCall(context, assistantMessage, preparation, executed, options, signal);
}
async function executePreparedToolCall(
prepared: PreparedToolCall,
signal: AbortSignal | undefined,
emit: AgentEventSink,
onUpdate: ToolUpdateSink,
): Promise<ExecutedToolCallOutcome> {
const updateEvents: Promise<void>[] = [];
let acceptingUpdates = true;
const startedAt = performance.now();
const elapsed = () => Math.round(performance.now() - startedAt);
try {
const result = await prepared.tool.execute(
@@ -689,28 +835,21 @@ async function executePreparedToolCall(
signal,
(partialResult) => {
if (!acceptingUpdates) return;
updateEvents.push(
Promise.resolve(
emit({
type: "tool_execution_update",
toolCallId: prepared.toolCall.id,
toolName: prepared.toolCall.name,
args: prepared.toolCall.arguments,
partialResult,
}),
),
);
updateEvents.push(Promise.resolve(onUpdate(partialResult)));
},
);
const durationMs = elapsed();
acceptingUpdates = false;
await Promise.all(updateEvents);
return { result, isError: false };
return { result, isError: result.isError === true, durationMs };
} catch (error) {
const durationMs = elapsed();
acceptingUpdates = false;
await Promise.all(updateEvents);
return {
result: createErrorToolResult(error instanceof Error ? error.message : String(error)),
isError: true,
durationMs,
};
} finally {
acceptingUpdates = false;
@@ -722,7 +861,7 @@ async function finalizeExecutedToolCall(
assistantMessage: AssistantMessage,
prepared: PreparedToolCall,
executed: ExecutedToolCallOutcome,
config: AgentLoopConfig,
config: ToolCallHooks,
signal: AbortSignal | undefined,
): Promise<FinalizedToolCallOutcome> {
let result = executed.result;
@@ -742,6 +881,9 @@ async function finalizeExecutedToolCall(
signal,
);
if (afterResult) {
// Structured content not replaced along with the content may no longer match it.
const structuredContent =
afterResult.structuredContent ?? (afterResult.content ? undefined : result.structuredContent);
result = {
...result,
content: afterResult.content ?? result.content,
@@ -749,6 +891,8 @@ async function finalizeExecutedToolCall(
usage: afterResult.usage ?? result.usage,
terminate: afterResult.terminate ?? result.terminate,
};
if (structuredContent === undefined) delete result.structuredContent;
else result.structuredContent = structuredContent;
isError = afterResult.isError ?? isError;
}
} catch (error) {
@@ -761,6 +905,7 @@ async function finalizeExecutedToolCall(
toolCall: prepared.toolCall,
result,
isError,
durationMs: executed.durationMs,
};
}
@@ -778,6 +923,7 @@ async function emitToolExecutionEnd(finalized: FinalizedToolCallOutcome, emit: A
toolName: finalized.toolCall.name,
result: finalized.result,
isError: finalized.isError,
...(finalized.durationMs === undefined ? {} : { durationMs: finalized.durationMs }),
});
}
@@ -791,8 +937,8 @@ function createToolResultMessage(finalized: FinalizedToolCallOutcome): ToolResul
content: finalized.result.content ?? [],
details: finalized.result.details,
usage: finalized.result.usage,
...(finalized.result.addedToolNames?.length ? { addedToolNames: finalized.result.addedToolNames } : {}),
isError: finalized.isError,
...(finalized.durationMs === undefined ? {} : { durationMs: finalized.durationMs }),
timestamp: Date.now(),
};
}

103
third_party/pi/packages/agent/src/agent.ts vendored
View File

@@ -1,11 +1,15 @@
import type {
ImageContent,
Message,
Model,
SimpleStreamOptions,
TextContent,
ThinkingBudgets,
Transport,
import {
createInitialSystemMessage,
getCurrentSystemMessage,
getCurrentSystemPrompt,
type ImageContent,
type Message,
type Model,
type SimpleStreamOptions,
type TextContent,
type ThinkingBudgets,
type Transport,
toToolDeclaration,
} from "@earendil-works/pi-ai";
import { runAgentLoop, runAgentLoopContinue } from "./agent-loop.ts";
import { getDefaultStreamFn } from "./stream-fn.ts";
@@ -21,9 +25,10 @@ import type {
AgentTool,
BeforeToolCallContext,
BeforeToolCallResult,
FinishTurn,
PrepareNextTurnContext,
PrepareRequest,
QueueMode,
ShouldStopAfterTurnContext,
StreamFn,
ToolExecutionMode,
} from "./types.ts";
@@ -32,7 +37,11 @@ export type { QueueMode } from "./types.ts";
function defaultConvertToLlm(messages: AgentMessage[]): Message[] {
return messages.filter(
(message) => message.role === "user" || message.role === "assistant" || message.role === "toolResult",
(message) =>
message.role === "system" ||
message.role === "user" ||
message.role === "assistant" ||
message.role === "toolResult",
);
}
@@ -65,14 +74,21 @@ type MutableAgentState = Omit<AgentState, "isStreaming" | "streamingMessage" | "
errorMessage?: string;
};
function createMutableAgentState(
initialState?: Partial<Omit<AgentState, "pendingToolCalls" | "isStreaming" | "streamingMessage" | "errorMessage">>,
): MutableAgentState {
/** Initial state for {@link Agent}. `systemPrompt` and `tools` become the leading system message unless `messages` already starts with one. */
export type AgentInitialState = Partial<
Omit<AgentState, "pendingToolCalls" | "isStreaming" | "streamingMessage" | "errorMessage">
>;
function createMutableAgentState(initialState?: AgentInitialState): MutableAgentState {
let tools = initialState?.tools?.slice() ?? [];
let messages = initialState?.messages?.slice() ?? [];
const initialMessage = createInitialSystemMessage(initialState?.systemPrompt, tools.map(toToolDeclaration));
if (messages[0]?.role !== "system" && initialMessage) messages.unshift(initialMessage);
return {
systemPrompt: initialState?.systemPrompt ?? "",
get systemPrompt() {
return getCurrentSystemPrompt(messages);
},
model: initialState?.model ?? DEFAULT_MODEL,
thinkingLevel: initialState?.thinkingLevel ?? "off",
get tools() {
@@ -96,16 +112,18 @@ function createMutableAgentState(
/** Options for constructing an {@link Agent}. */
export interface AgentOptions {
initialState?: Partial<Omit<AgentState, "pendingToolCalls" | "isStreaming" | "streamingMessage" | "errorMessage">>;
initialState?: AgentInitialState;
convertToLlm?: (messages: AgentMessage[]) => Message[] | Promise<Message[]>;
transformContext?: (messages: AgentMessage[], signal?: AbortSignal) => Promise<AgentMessage[]>;
streamFn: StreamFn;
getApiKey?: (provider: string) => Promise<string | undefined> | string | undefined;
onPayload?: SimpleStreamOptions["onPayload"];
onResponse?: SimpleStreamOptions["onResponse"];
onProviderStreamEvent?: SimpleStreamOptions["onProviderStreamEvent"];
beforeToolCall?: (context: BeforeToolCallContext, signal?: AbortSignal) => Promise<BeforeToolCallResult | undefined>;
afterToolCall?: (context: AfterToolCallContext, signal?: AbortSignal) => Promise<AfterToolCallResult | undefined>;
shouldStopAfterTurn?: (context: ShouldStopAfterTurnContext, signal?: AbortSignal) => boolean | Promise<boolean>;
finishTurn?: FinishTurn;
prepareRequest?: PrepareRequest;
prepareNextTurn?: (
signal?: AbortSignal,
) => Promise<AgentLoopTurnUpdate | undefined> | AgentLoopTurnUpdate | undefined;
@@ -138,19 +156,16 @@ class PendingMessageQueue {
return this.messages.length > 0;
}
drain(): AgentMessage[] {
if (this.mode === "all") {
const drained = this.messages.slice();
this.messages = [];
return drained;
}
peek(): AgentMessage[] {
if (this.mode === "all") return this.messages.slice();
const first = this.messages[0];
if (!first) {
return [];
}
this.messages = this.messages.slice(1);
return [first];
return first ? [first] : [];
}
drain(): AgentMessage[] {
const drained = this.peek();
this.messages = this.messages.slice(drained.length);
return drained;
}
clear(): void {
@@ -182,6 +197,7 @@ export class Agent {
public getApiKey?: (provider: string) => Promise<string | undefined> | string | undefined;
public onPayload?: SimpleStreamOptions["onPayload"];
public onResponse?: SimpleStreamOptions["onResponse"];
public onProviderStreamEvent?: SimpleStreamOptions["onProviderStreamEvent"];
public beforeToolCall?: (
context: BeforeToolCallContext,
signal?: AbortSignal,
@@ -190,10 +206,8 @@ export class Agent {
context: AfterToolCallContext,
signal?: AbortSignal,
) => Promise<AfterToolCallResult | undefined>;
public shouldStopAfterTurn?: (
context: ShouldStopAfterTurnContext,
signal?: AbortSignal,
) => boolean | Promise<boolean>;
public finishTurn?: FinishTurn;
public prepareRequest?: PrepareRequest;
public prepareNextTurn?: (
signal?: AbortSignal,
) => Promise<AgentLoopTurnUpdate | undefined> | AgentLoopTurnUpdate | undefined;
@@ -223,9 +237,11 @@ export class Agent {
this.getApiKey = runtimeOptions.getApiKey;
this.onPayload = runtimeOptions.onPayload;
this.onResponse = runtimeOptions.onResponse;
this.onProviderStreamEvent = runtimeOptions.onProviderStreamEvent;
this.beforeToolCall = runtimeOptions.beforeToolCall;
this.afterToolCall = runtimeOptions.afterToolCall;
this.shouldStopAfterTurn = runtimeOptions.shouldStopAfterTurn;
this.finishTurn = runtimeOptions.finishTurn;
this.prepareRequest = runtimeOptions.prepareRequest;
this.prepareNextTurn = runtimeOptions.prepareNextTurn;
this.prepareNextTurnWithContext = runtimeOptions.prepareNextTurnWithContext;
this.steeringQueue = new PendingMessageQueue(runtimeOptions.steeringMode ?? "one-at-a-time");
@@ -310,6 +326,12 @@ export class Agent {
return this.steeringQueue.hasItems() || this.followUpQueue.hasItems();
}
/** Preview the messages selected for the next turn without consuming them. */
peekQueuedMessages(): AgentMessage[] {
const steering = this.steeringQueue.peek();
return steering.length > 0 ? steering : this.followUpQueue.peek();
}
/** Active abort signal for the current run, if any. */
get signal(): AbortSignal | undefined {
return this.activeRun?.abortController.signal;
@@ -329,13 +351,14 @@ export class Agent {
return this.activeRun?.promise ?? Promise.resolve();
}
/** Clear transcript state, runtime state, and queued messages. */
/** Clear conversation state and queues while retaining the replayed prompt/tool baseline. */
reset(): void {
if (this.activeRun) {
throw new Error("Agent is already processing. Wait for completion before resetting.");
}
this._state.messages = [];
const baseline = getCurrentSystemMessage(this._state.messages);
this._state.messages = baseline ? [baseline] : [];
this._state.isStreaming = false;
this._state.streamingMessage = undefined;
this._state.pendingToolCalls = new Set<string>();
@@ -364,7 +387,7 @@ export class Agent {
}
const lastMessage = this._state.messages[this._state.messages.length - 1];
if (!lastMessage) {
if (!lastMessage || this._state.messages.every((message) => message.role === "system")) {
throw new Error("No messages to continue from");
}
@@ -436,7 +459,6 @@ export class Agent {
private createContextSnapshot(): AgentContext {
return {
systemPrompt: this._state.systemPrompt,
messages: this._state.messages.slice(),
tools: this._state.tools.slice(),
};
@@ -444,22 +466,21 @@ export class Agent {
private createLoopConfig(options: { skipInitialSteeringPoll?: boolean } = {}): AgentLoopConfig {
let skipInitialSteeringPoll = options.skipInitialSteeringPoll === true;
const shouldStopAfterTurn = this.shouldStopAfterTurn;
return {
model: this._state.model,
reasoning: this._state.thinkingLevel === "off" ? undefined : this._state.thinkingLevel,
sessionId: this.sessionId,
onPayload: this.onPayload,
onResponse: this.onResponse,
onProviderStreamEvent: this.onProviderStreamEvent,
transport: this.transport,
thinkingBudgets: this.thinkingBudgets,
maxRetryDelayMs: this.maxRetryDelayMs,
toolExecution: this.toolExecution,
beforeToolCall: this.beforeToolCall,
afterToolCall: this.afterToolCall,
shouldStopAfterTurn: shouldStopAfterTurn
? async (context) => await shouldStopAfterTurn(context, this.signal)
: undefined,
finishTurn: this.finishTurn,
prepareRequest: this.prepareRequest,
prepareNextTurn:
this.prepareNextTurnWithContext || this.prepareNextTurn
? async (context) => {

622
third_party/pi/packages/agent/src/harness/agent-harness.ts vendored
View File

@@ -1,622 +0,0 @@
import type { JsonRepresentation } from "@earendil-works/chord";
import type {
Api,
AssistantMessage,
AssistantMessageEvent,
AssistantMessageFrame,
DeferredHandle,
ImageContent,
Message,
Model,
Models,
RetryPolicy,
ToolResultMessage,
Usage,
} from "@earendil-works/pi-ai";
import type { AgentMessage, AgentToolResult, QueueMode, ThinkingLevel } from "../types.ts";
import type { BranchPreparation, BranchSummaryResult } from "./compaction/branch-summarization.ts";
import type { CompactionPreparation, CompactionSettings, CompactResult } from "./compaction/compaction.ts";
import type { Context } from "./context.ts";
import type {
Closed,
InvalidMessage,
InvalidNavigation,
LaneBusy,
NoActiveOperation,
NothingToCompact,
NothingToResume,
OperationMismatch,
Result,
UnknownSkill,
UnknownTarget,
UnknownTemplate,
} from "./result.ts";
export {
Closed,
HarnessClosed,
HarnessFault,
InvalidLane,
InvalidMessage,
InvalidNavigation,
LaneBusy,
NoActiveOperation,
NoActiveRun,
NothingToCompact,
NothingToResume,
OperationMismatch,
UnknownSkill,
UnknownTarget,
UnknownTemplate,
} from "./result.ts";
export { SliceNotImplemented } from "./runtime/types.ts";
import { createAgentHarness } from "./runtime/harness.ts";
import type {
BranchScan,
Entry,
EntryProjector,
JsonValue,
LaneConfiguration,
OperationError,
OperationResultRecord,
Session,
SessionStats,
SettledAssistantMessage,
UsageRow,
} from "./session/types.ts";
import type {
AgentHarnessResources,
AgentHarnessStreamOptions,
AgentHarnessStreamOptionsPatch,
AgentHarnessTool,
PromptTemplate,
Skill,
} from "./types.ts";
/** Convenience-only suspended run observation, constructed when M8 exposes public drive. */
export interface SuspendedRun {
operationId: string;
status: "suspended";
deferred: DeferredHandle;
}
export type RunResult = Result<
OperationResultRecord | SuspendedRun,
LaneBusy | InvalidMessage | UnknownSkill | UnknownTemplate | Closed
>;
export type CompactionResult = Result<
{ compaction: OperationResultRecord; run?: OperationResultRecord | SuspendedRun },
LaneBusy | NothingToCompact | Closed
>;
export type NavigationResult = Result<
{ navigation: OperationResultRecord; run?: OperationResultRecord | SuspendedRun },
LaneBusy | InvalidNavigation | UnknownTarget | Closed
>;
export type ResumeResult = Result<OperationResultRecord | SuspendedRun, NothingToResume | Closed>;
export type QueueResult = Result<{ entryId: string }, InvalidMessage | Closed>;
export type CancelQueuedResult = Result<{ kind: "cancelled" | "already_consumed" | "not_found" }, Closed>;
export type AbortResult = Result<
{ operationId: string; steer: AgentMessage[]; followUp: AgentMessage[] },
NoActiveOperation | Closed
>;
export type RecordUsageResult = Result<{ usageId: string }, Closed>;
export interface NavigateOptions {
summarize?: boolean;
label?: string;
customInstructions?: string;
}
export type OperationRequest =
| { kind: "prompt"; operationId?: string; prompt: string; images?: ImageContent[] }
| { kind: "prompt"; operationId?: string; prompt: AgentMessage | AgentMessage[]; images?: never }
| { kind: "skill"; operationId?: string; name: string; additionalInstructions?: string }
| { kind: "prompt_template"; operationId?: string; name: string; args?: string[] }
| { kind: "compaction"; operationId?: string; customInstructions?: string }
| { kind: "navigation"; operationId?: string; targetId: string | null; options?: NavigateOptions };
export interface OperationAdmission {
operationId: string;
kind: "run" | "compaction" | "navigation";
startedAt: number;
}
export type OperationAdmissionError =
| LaneBusy
| InvalidMessage
| UnknownSkill
| UnknownTemplate
| NothingToCompact
| InvalidNavigation
| UnknownTarget
| Closed;
export type OperationAdmissionResult = Result<OperationAdmission, OperationAdmissionError>;
export interface DriveOptions {
operationId: string;
waitForRetry?: boolean;
pollDeferred?: boolean;
}
export interface ModelIdentity {
provider: string;
modelId: string;
}
export type OperationStatus = "running" | "open" | "aborting";
export interface CurrentOperationInfo {
id: string;
kind: "run" | "compaction" | "navigation";
startedAt: number;
status: OperationStatus;
capturedModel?: ModelIdentity;
}
export interface LaneExecutionInfo {
lane: string;
tipId: string | null;
configuredModel: ModelIdentity;
current: CurrentOperationInfo | null;
lastOperationId: string | null;
}
export type DriveOutcome =
| { kind: "settled"; outcome: OperationResultRecord }
| { kind: "waiting"; operationId: string; reason: "retry"; notBefore: number }
| { kind: "waiting"; operationId: string; reason: "deferred"; deferred: DeferredHandle };
export type DriveResult = Result<DriveOutcome, OperationMismatch | Closed>;
export type AbortRequestResult = Result<
{
operationId: string;
newlyRequested: boolean;
steer: AgentMessage[];
followUp: AgentMessage[];
},
OperationMismatch | Closed
>;
export interface WatchHandle<T> {
snapshot: T;
start(listener: EventListener): void;
resnapshot(context: Context): Promise<T>;
unsubscribe(): void;
}
export interface LaneInfo {
name: string;
tipId: string | null;
operation: CurrentOperationInfo | null;
}
export type LaneSnapshotTool =
| {
status: "running";
toolCallId: string;
toolName: string;
args: unknown;
result?: AgentToolResult<unknown>;
}
| {
status: "settled";
toolCallId: string;
toolName: string;
args: unknown;
result: AgentToolResult<unknown>;
isError: boolean;
};
export interface OpenOperation {
lane: string;
operationId: string;
kind: "run" | "compaction" | "navigation";
startedAt: number;
aborting?: true;
}
export type LaneQueuedItem =
| {
entryId: string;
kind: "steer" | "followUp" | "nextRun" | "write";
type: "message";
message: AgentMessage;
}
| { entryId: string; kind: "write"; type: "custom"; customType: string; data?: JsonValue };
export interface LaneSnapshot {
lane: string;
transcript: Entry[];
tipId: string | null;
lastResult?: OperationResultRecord;
configuration: LaneConfiguration;
stats: SessionStats;
operation: null | {
id: string;
kind: "run" | "compaction" | "navigation";
startedAt: number;
fromTipId: string | null;
status: OperationStatus;
retry?: { attempt: number; maxAttempts: number; nextAttemptAt: number };
deferred?: { handle: DeferredHandle; poll: number };
streamingMessage?: AssistantMessage;
runningTools: LaneSnapshotTool[];
};
queues: LaneQueuedItem[];
faulted: boolean;
}
export interface SessionSnapshot {
lanes: LaneInfo[];
faulted: boolean;
}
export type HarnessEventPayload =
| { type: "run_start"; runId: string; startedAt: number }
| { type: "run_resume"; runId: string }
| { type: "run_suspend"; runId: string; reason: "deferred"; deferred: DeferredHandle; poll: number }
| { type: "operation_abort"; operationId: string; steer: AgentMessage[]; followUp: AgentMessage[] }
| ({ type: "run_end"; runId: string; fromTipId: string | null; tipId: string | null; endedAt: number } & (
| { status: "completed" | "aborted"; error?: never }
| { status: "failed"; error: OperationError }
))
| { type: "fault"; code: string; message: string }
| ({ type: "handler_error"; error: string; stack?: string } & (
| { kind: "hook"; hook: string }
| { kind: "event"; event: string }
))
| { type: "turn_start"; runId: string; turnId: string }
| {
type: "turn_end";
runId: string;
turnId: string;
message: AssistantMessage;
toolResults: ToolResultMessage[];
}
| {
type: "retry_scheduled";
runId: string;
step: string;
attempt: number;
maxAttempts: number;
delayMs: number;
notBefore: number;
errorMessage: string;
}
| { type: "retry_start"; runId: string; step: string; attempt: number }
| {
type: "retry_end";
runId: string;
step: string;
attempt: number;
success: boolean;
finalError?: string;
}
| { type: "message_start"; runId?: string; message: AgentMessage }
| {
type: "message_update";
runId: string;
message: AgentMessage;
event: AssistantMessageEvent;
frame?: AssistantMessageFrame;
}
| { type: "message_end"; runId?: string; message: AgentMessage; entryId?: string }
| {
type: "tool_start";
runId: string;
turnId: string;
toolCallId: string;
toolName: string;
args: unknown;
}
| {
type: "tool_update";
runId: string;
turnId: string;
toolCallId: string;
toolName: string;
partialResult: AgentToolResult<unknown>;
}
| {
type: "tool_end";
runId: string;
turnId: string;
toolCallId: string;
toolName: string;
result: AgentToolResult<unknown>;
isError: boolean;
terminate: boolean;
}
| { type: "entry_added"; entry: Entry }
| { type: "queue_update"; queues: LaneQueuedItem[] }
| ({ type: "value_update" } & (
| { value: "session_name"; name: string | undefined }
| { value: "entry_label"; targetId: string; label: string | undefined }
))
| ({ type: "config_update" } & (
| {
property: "model";
value: { provider: string; modelId: string };
previous: unknown;
}
| { property: "thinkingLevel"; value: ThinkingLevel; previous: ThinkingLevel }
| { property: "activeTools"; value: string[]; previous: string[] }
| { property: "tools" | "resources" }
| {
property: "streamOptions";
value: AgentHarnessStreamOptions;
previous: AgentHarnessStreamOptions;
}
| { property: "retryPolicy"; value: RetryPolicy; previous: RetryPolicy }
| { property: "compactionSettings"; value: CompactionSettings; previous: CompactionSettings }
| { property: "steeringMode"; value: QueueMode; previous: QueueMode }
| { property: "followUpMode"; value: QueueMode; previous: QueueMode }
))
| {
type: "compaction_start";
runId: string;
reason: "manual" | "threshold" | "overflow";
startedAt: number;
}
| ({ type: "compaction_end"; runId: string; reason: "manual" | "threshold" | "overflow"; endedAt: number } & (
| { status: "completed"; entryId: string; error?: never }
| { status: "declined" | "aborted"; entryId?: never; error?: never }
| { status: "failed"; entryId?: never; error: OperationError }
))
| { type: "navigation_start"; runId: string; targetId: string | null; startedAt: number }
| ({ type: "navigation_end"; runId: string; fromTipId: string | null; tipId: string | null; endedAt: number } & (
| { status: "completed" | "declined" | "aborted"; error?: never }
| { status: "failed"; error: OperationError }
))
| { type: "lane_created"; at: string | null }
| { type: "usage"; lane: string; row: UsageRow; totals: Usage };
export type SpecialEventPayload = Extract<
HarnessEventPayload,
{ type: "fault" | "value_update" | "usage" | "config_update" | "handler_error" }
>;
export type LaneEventPayload = Exclude<HarnessEventPayload, SpecialEventPayload>;
export type ConfigEventPayload = Extract<HarnessEventPayload, { type: "config_update" }>;
export type LaneConfigEventPayload = Extract<
ConfigEventPayload,
{ property: "model" | "thinkingLevel" | "activeTools" }
>;
export type GlobalConfigEventPayload = Exclude<ConfigEventPayload, LaneConfigEventPayload>;
export type HandlerErrorPayload = Extract<HarnessEventPayload, { type: "handler_error" }>;
export type HarnessEvent =
| (LaneEventPayload & { lane: string; recovery?: true })
| (LaneConfigEventPayload & { lane: string; recovery?: true })
| (Extract<HarnessEventPayload, { type: "fault" | "value_update" }> & {
lane?: never;
recovery?: never;
})
| (Extract<HarnessEventPayload, { type: "usage" }> & { recovery?: never })
| (GlobalConfigEventPayload & { lane?: never; recovery?: never })
| (HandlerErrorPayload & ({ lane: string; recovery?: true } | { lane?: never; recovery?: never }));
type LaneWatchSourceEvent =
| Exclude<
HarnessEvent,
| { type: "handler_error" | "turn_start" | "turn_end" | "value_update" | "lane_created" | "message_update" }
| ({ type: "config_update" } & { property: string })
>
| Extract<HarnessEvent, { type: "config_update"; property: "model" | "thinkingLevel" | "activeTools" }>
| Omit<Extract<HarnessEvent, { type: "message_update" }>, "event">;
/** Strict-JSON snapshot representation published to remote transcript consumers. */
export type LaneTranscriptSnapshot = JsonRepresentation<LaneSnapshot>;
/** Reducer-relevant strict-JSON Harness events published to remote transcript consumers. */
export type LaneWatchEvent = JsonRepresentation<LaneWatchSourceEvent>;
export type HarnessEventType = HarnessEvent["type"];
export type EventListener<TEvent extends HarnessEvent = HarnessEvent> = (
event: TEvent,
context: Context,
) => void | Promise<void>;
export interface Events {
on<TType extends HarnessEventType>(
type: TType,
listener: EventListener<Extract<HarnessEvent, { type: TType }>>,
): () => void;
}
export type Resources = AgentHarnessResources<Skill, PromptTemplate>;
type VoidHookResult = ReturnType<() => void>;
export interface HookMap {
before_run: {
event: { prompt: AgentMessage[]; resources: Resources };
result: { messages?: AgentMessage[] } | undefined;
};
before_drive: {
event: { operation: "run" | "compaction" | "navigation" };
result: VoidHookResult;
};
before_run_end: {
event: { runId: string; messages: AgentMessage[] };
result: { followUp?: string } | undefined;
};
transform_context: {
event: { messages: AgentMessage[]; systemPrompt: string };
result: { messages?: AgentMessage[]; systemPrompt?: string } | undefined;
};
before_request: {
event: {
model: Model<Api>;
step: "assistant" | "deferred" | "compaction" | "branch_summary";
attempt: number;
streamOptions: AgentHarnessStreamOptions;
};
result: { streamOptions?: AgentHarnessStreamOptionsPatch } | undefined;
};
before_payload: {
event: { model: Model<Api>; payload: unknown };
result: { payload: unknown } | undefined;
};
after_response: {
event: { status?: number; headers?: Record<string, string>; message: SettledAssistantMessage };
result: { message?: SettledAssistantMessage } | undefined;
};
before_tool: {
event: { toolCallId: string; toolName: string; args: Record<string, JsonValue> };
result: { args?: Record<string, JsonValue>; block?: { reason: string; terminate?: boolean } } | undefined;
};
after_tool: {
event: {
toolCallId: string;
toolName: string;
args: Record<string, JsonValue>;
content: AgentToolResult<unknown>["content"];
details?: JsonValue;
isError: boolean;
usage?: Usage;
};
result:
| {
content?: AgentToolResult<unknown>["content"];
details?: JsonValue;
isError?: boolean;
usage?: Usage;
terminate?: boolean;
}
| undefined;
};
before_compaction: {
event: {
reason: "manual" | "threshold" | "overflow";
preparation: CompactionPreparation;
customInstructions?: string;
};
result: { decline?: boolean; compaction?: CompactResult } | undefined;
};
before_navigation: {
event: { targetId: string; preparation: BranchPreparation; customInstructions?: string };
result: { decline?: boolean; summary?: BranchSummaryResult } | undefined;
};
}
export type HookName = keyof HookMap;
export type HookInvocation<TName extends HookName> = HookMap[TName]["event"] & {
lane: string;
runId: string;
};
export type HookHandler<TName extends HookName> = (
event: HookInvocation<TName>,
context: Context,
) => Promise<HookMap[TName]["result"]> | HookMap[TName]["result"];
export interface Hooks {
on<TName extends HookName>(name: TName, handler: HookHandler<TName>, options?: { id?: string }): () => void;
}
export type { EntryProjector } from "./session/types.ts";
export interface AgentHarnessOptions<TContext extends object | undefined = object | undefined> {
session: Session;
models: Models;
model: Model<Api>;
thinkingLevel?: ThinkingLevel;
activeToolNames?: string[];
tools?: AgentHarnessTool<TContext>[];
toolContext?: TContext | ((context: Context) => TContext | Promise<TContext>);
systemPrompt?: string | ((toolContext: TContext, context: Context) => string | Promise<string>);
resources?: Resources;
streamOptions?: AgentHarnessStreamOptions;
retry?: RetryPolicy;
compaction?: CompactionSettings;
steeringMode?: QueueMode;
followUpMode?: QueueMode;
toolExecution?: "sequential" | "parallel";
toProviderMessages?: (messages: AgentMessage[], context: Context) => Message[] | Promise<Message[]>;
entryProjectors?: Record<string, EntryProjector>;
}
export interface AgentLane {
readonly name: string;
getTipId(context: Context): Promise<string | null>;
findEntries(query: BranchScan | undefined, context: Context): Promise<Entry[]>;
findEntry(query: BranchScan | undefined, context: Context): Promise<Entry | undefined>;
appendMessage(message: AgentMessage, context: Context): Promise<string>;
appendCustomEntry(customType: string, data: JsonValue | undefined, context: Context): Promise<string>;
getResult(operationId: string, context: Context): Promise<OperationResultRecord | undefined>;
accept(request: OperationRequest, context: Context): Promise<OperationAdmissionResult>;
drive(options: DriveOptions, context: Context): Promise<DriveResult>;
requestAbort(operationId: string, context: Context): Promise<AbortRequestResult>;
inspectExecution(context: Context): Promise<LaneExecutionInfo>;
prompt(text: string, images: ImageContent[] | undefined, context: Context): Promise<RunResult>;
prompt(message: AgentMessage | AgentMessage[], context: Context): Promise<RunResult>;
skill(name: string, additionalInstructions: string | undefined, context: Context): Promise<RunResult>;
promptFromTemplate(name: string, args: string[] | undefined, context: Context): Promise<RunResult>;
compact(options: { customInstructions?: string } | undefined, context: Context): Promise<CompactionResult>;
navigateTree(
targetId: string | null,
options: NavigateOptions | undefined,
context: Context,
): Promise<NavigationResult>;
resume(context: Context): Promise<ResumeResult>;
abort(context: Context): Promise<AbortResult>;
steer(message: string | AgentMessage, images: ImageContent[] | undefined, context: Context): Promise<QueueResult>;
followUp(message: string | AgentMessage, images: ImageContent[] | undefined, context: Context): Promise<QueueResult>;
nextRun(message: string | AgentMessage, images: ImageContent[] | undefined, context: Context): Promise<QueueResult>;
cancelQueued(entryId: string, context: Context): Promise<CancelQueuedResult>;
recordUsage(
usage: Usage,
options: { entryId?: string; details?: JsonValue } | undefined,
context: Context,
): Promise<RecordUsageResult>;
waitForIdle(context: Context): Promise<void>;
runWhenIdle(callback: (context: Context) => void | Promise<void>, context: Context): Promise<void>;
getModel(context: Context): Promise<Model<Api> | undefined>;
setModel(model: ModelIdentity, context: Context): Promise<void>;
getThinkingLevel(context: Context): Promise<ThinkingLevel>;
setThinkingLevel(level: ThinkingLevel, context: Context): Promise<void>;
getActiveTools(context: Context): Promise<string[]>;
setActiveTools(names: string[], context: Context): Promise<void>;
watch(context: Context): Promise<WatchHandle<LaneSnapshot>>;
}
export interface AcquireLaneOptions {
createAt?: string | null;
}
export interface AgentHarness<TContext extends object | undefined = object | undefined> {
lane(name: string, context: Context): Promise<AgentLane>;
lane(name: string, options: AcquireLaneOptions, context: Context): Promise<AgentLane>;
lanes(context: Context): Promise<LaneInfo[]>;
getName(context: Context): Promise<string | undefined>;
setName(name: string | undefined, context: Context): Promise<void>;
getLabel(targetId: string, context: Context): Promise<string | undefined>;
setLabel(targetId: string, label: string | undefined, context: Context): Promise<void>;
getTools(context: Context): Promise<AgentHarnessTool<TContext>[]>;
setTools(tools: AgentHarnessTool<TContext>[], context: Context): Promise<void>;
getResources(context: Context): Promise<Resources>;
setResources(resources: Resources, context: Context): Promise<void>;
getStreamOptions(context: Context): Promise<AgentHarnessStreamOptions>;
setStreamOptions(options: AgentHarnessStreamOptions, context: Context): Promise<void>;
getRetryPolicy(context: Context): Promise<RetryPolicy>;
setRetryPolicy(policy: RetryPolicy, context: Context): Promise<void>;
getCompactionSettings(context: Context): Promise<CompactionSettings>;
setCompactionSettings(settings: CompactionSettings, context: Context): Promise<void>;
getSteeringMode(context: Context): Promise<QueueMode>;
setSteeringMode(mode: QueueMode, context: Context): Promise<void>;
getFollowUpMode(context: Context): Promise<QueueMode>;
setFollowUpMode(mode: QueueMode, context: Context): Promise<void>;
watchSession(context: Context): Promise<WatchHandle<SessionSnapshot>>;
readonly hooks: Hooks;
readonly events: Events;
close(context: Context): Promise<void>;
}
export interface AgentHarnessConstructor {
create<TContext extends object | undefined = object | undefined>(
options: AgentHarnessOptions<TContext>,
context: Context,
): Promise<{ harness: AgentHarness<TContext>; open: OpenOperation[] }>;
}
/** Runtime constructor for attaching the durable harness to one open session. */
export const AgentHarness = { create: createAgentHarness } satisfies AgentHarnessConstructor;

Some files were not shown because too many files have changed in this diff Show More