Squashed 'src/pi/' content from commit d981de122

git-subtree-dir: src/pi
git-subtree-split: d981de1229ef899957bbe968bc8dcda02a21f477
This commit is contained in:
2026-09-13 07:04:45 -04:00
commit 25b62d1feb
1669 changed files with 397627 additions and 0 deletions

View File

@@ -0,0 +1,172 @@
import { type Static, Type } from "typebox";
import type { Context } from "../context.ts";
import type { AgentHarnessTool } from "../types.ts";
import { getOrThrow } from "../types.ts";
import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult } from "../utils/truncate.ts";
import type { ExecutionToolContext } from "./tool-context.ts";
const MAX_TIMEOUT_SECONDS = 2_147_483_647 / 1000;
/**
* Coalescing cadence for live updates. The env owns capping and coalescing now
* (`executionenv.md` §3), so bash no longer has an update throttle or a
* checkpoint interval of its own.
*/
const BASH_UPDATE_INTERVAL_MS = 100;
const bashSchema = Type.Object({
command: Type.String({ description: "Bash command to execute" }),
timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })),
});
export type BashToolInput = Static<typeof bashSchema>;
export interface BashToolDetails {
truncation?: TruncationResult;
/** Set by the exec env, on the machine the command ran on. */
spillPath?: string;
}
export interface BashExecution {
command: string;
cwd: string;
env: Record<string, string>;
inheritEnv: boolean;
}
export type BashPrepare<TContext extends ExecutionToolContext = ExecutionToolContext> = (
execution: BashExecution,
toolContext: TContext,
context: Context,
) => void | Promise<void>;
export interface BashToolOptions<TContext extends ExecutionToolContext = ExecutionToolContext> {
commandPrefix?: string;
prepare?: BashPrepare<TContext>;
}
function validateTimeout(timeout: number | undefined): void {
if (timeout === undefined) return;
if (!Number.isFinite(timeout) || timeout <= 0) {
throw new Error("Invalid timeout: must be a finite number of seconds");
}
if (timeout > MAX_TIMEOUT_SECONDS) {
throw new Error(`Invalid timeout: maximum is ${MAX_TIMEOUT_SECONDS} seconds`);
}
}
export function createBashTool<TContext extends ExecutionToolContext = ExecutionToolContext>(
options?: BashToolOptions<TContext>,
): AgentHarnessTool<TContext, typeof bashSchema, BashToolDetails | undefined> {
return {
name: "bash",
label: "bash",
description: `Execute a bash command in the current working directory. Returns stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`,
parameters: bashSchema,
async execute(_toolCallId, { command, timeout }, onUpdate, toolContext, _invocation, context) {
validateTimeout(timeout);
const { env } = toolContext;
const execution: BashExecution = {
command: options?.commandPrefix ? `${options.commandPrefix}\n${command}` : command,
cwd: env.cwd,
env: {},
inheritEnv: true,
};
await options?.prepare?.(execution, toolContext, context);
// The env caps, coalesces and spills at the source. bash keeps a mirror
// only to build the settled result; it never truncates and never chooses
// a spill path.
let view = "";
let truncation: TruncationResult | undefined;
let spillPath: string | undefined;
const publish = (checkpoint: boolean): void => {
onUpdate(
{
content: [{ type: "text" as const, text: view }],
details: { truncation, spillPath },
},
checkpoint ? { checkpoint: true } : undefined,
);
};
onUpdate({ content: [], details: undefined });
const result = await env.exec(
execution.command,
{
cwd: execution.cwd,
env: execution.env,
inheritEnv: execution.inheritEnv,
timeout,
capture: {
limits: { maxBytes: DEFAULT_MAX_BYTES, maxLines: DEFAULT_MAX_LINES, retain: "tail" },
intervalMs: BASH_UPDATE_INTERVAL_MS,
spill: true,
},
onUpdate: (update) => {
switch (update.kind) {
case "chunk":
view += update.text;
break;
case "snapshot":
view = update.output.text;
truncation = update.output.truncation;
spillPath = update.output.spillPath;
break;
case "counters":
truncation = update.truncation;
break;
}
// A snapshot is the durable-worthy state: it is the only kind that
// can express eviction, so it is where a crash loses least.
publish(update.kind === "snapshot");
},
},
context,
);
if (!result.ok) {
const error = result.error;
const status =
error.code === "timeout"
? `Command timed out after ${timeout} seconds`
: error.code === "aborted"
? "Command aborted"
: error.message;
throw new Error(view ? `${view}\n\n${status}` : status, { cause: error });
}
const captured = result.value.output;
view = captured.text;
truncation = captured.truncation;
spillPath = captured.spillPath;
let outputText = captured.text;
let details: BashToolDetails | undefined;
if (captured.truncation.truncated) {
details = { truncation: captured.truncation, spillPath };
const startLine = captured.truncation.totalLines - captured.truncation.outputLines + 1;
const endLine = captured.truncation.totalLines;
if (captured.truncation.lastLinePartial) {
// One line alone exceeded the byte cap. Line numbers would be
// misleading here, so report the line's own size instead — and only
// the env can measure it, since it holds the raw buffer.
const lastLineSize = formatSize(captured.lastLineBytes ?? captured.truncation.outputBytes);
outputText += `\n\n[Showing last ${formatSize(captured.truncation.outputBytes)} of line ${endLine} (line is ${lastLineSize}). Full output: ${spillPath}]`;
} else if (captured.truncation.truncatedBy === "lines") {
outputText += `\n\n[Showing lines ${startLine}-${endLine} of ${captured.truncation.totalLines}. Full output: ${spillPath}]`;
} else {
outputText += `\n\n[Showing lines ${startLine}-${endLine} of ${captured.truncation.totalLines} (${formatSize(DEFAULT_MAX_BYTES)} limit). Full output: ${spillPath}]`;
}
}
if (result.value.exitCode !== 0) {
throw new Error(
`${outputText ? `${outputText}\n\n` : ""}Command exited with code ${result.value.exitCode}`,
);
}
return { content: [{ type: "text", text: outputText || "(no output)" }], details };
},
};
}

View File

@@ -0,0 +1,148 @@
# ExecutionEnv: bounded shell output
**Status:** implemented in production source. The prototype files beside this document are historical evidence; production lives in:
- `packages/agent/src/harness/utils/adaptive-publisher.ts`
- `packages/agent/src/harness/utils/output-capture.ts`
- `packages/agent/src/harness/env/nodejs.ts`
- `packages/agent/src/harness/tools/bash.ts`
The generic `ToolOutput` use of the same publisher remains in `04-tool-output`.
## 1. Problem
The old `Shell.exec()` accumulated complete strings inside `NodeExecutionEnv`:
```ts
stdout += chunk;
stderr += chunk;
```
Bash truncated only after those strings had already been built. `cat 1gb.txt` therefore materialized a gigabyte in the worker before any tool-level bound could help.
Spilling also belongs where bytes originate. If execution is remote, a spill on the worker is inaccessible to the model's `read` and `grep` tools, and creating it after transport would first send the complete gigabyte over the connection.
## 2. Boundary
`ExecutionEnv` now owns:
- a bounded head or tail view;
- complete byte and line totals;
- lazy source-local spill after the view first crosses its limits;
- persistent source-local spill with bounded write-stream backpressure;
- adaptive publication of the latest bounded state;
- a forced final publication before settlement.
It does not return or retain separate `stdout` and `stderr` values. Both pipes feed one arrival-ordered model-visible text view, matching bash and `ToolOutput`; preserving stream styling through tail eviction would require a segmented retained state that the Harness does not expose. Text is folded from updates, and `ShellExecResult` contains only exit and truncation/spill metadata.
Bash owns only command semantics and its model-visible footer. Its old rolling buffer, spill creation, 100 ms throttle, and full-output accumulation are gone. The existing two-second durable checkpoint request remains temporarily until `ToolOutput` owns durable cadence.
## 3. Contract
```ts
interface ShellOutputLimits {
maxBytes: number;
maxLines: number;
retain?: "head" | "tail";
}
interface ShellOutputCaptureOptions {
limits: ShellOutputLimits;
spill?: boolean;
}
type ShellOutputTruncation = Omit<TruncationResult, "content">;
interface ShellOutputMetadata {
truncation: ShellOutputTruncation;
spillPath?: string;
lastLineBytes?: number;
}
interface ShellOutputView extends ShellOutputMetadata {
text: string;
}
type ShellOutputUpdate =
| { kind: "replace"; output: ShellOutputView }
| { kind: "append"; text: string; metadata: ShellOutputMetadata }
| { kind: "slide"; drop: number; text: string; metadata: ShellOutputMetadata }
| { kind: "metadata"; metadata: ShellOutputMetadata };
interface ShellExecResult extends ShellOutputMetadata {
exitCode: number;
}
```
`drop` counts JavaScript string code units, matching `slice()`. Updates are ordered. A consumer applies them with `applyShellOutputUpdate()`.
A complete replacement establishes the initial state or recovers when no verified overlap exists. An append carries only a growing suffix. A slide drops a prefix and appends the new suffix. Metadata updates move totals or the spill path without resending text.
The compatibility `executeShellWithCapture()` helper still returns one bounded final view. Its `onChunk` callback receives initial, append, and slide text only; metadata and post-turnover replacements are not mislabeled as new bytes.
## 4. Adaptive publication
`AdaptivePublisher` keeps only the latest dirty state. Intermediate process writes never become a queue of output updates.
Policy is harness-global rather than per tool:
```ts
minIntervalMs = 100;
targetBytesPerSecond = 100 * 1024;
nextDelayMs = max(minIntervalMs, encodedUpdateBytes * 1000 / targetBytesPerSecond);
```
The first dirty state after idle is immediate. Writes received before the deadline collapse into the latest bounded state. One trailing timer guarantees eventual publication. Finalization bypasses the deadline once, still bounded by the retained cap.
The publisher commits its baseline before invoking the consumer. If a consumer applies an update and then throws, finalization cannot emit the same delta twice. The command fails with `callback_error`.
### Workload behavior
| workload | result |
| --- | --- |
| Finishes below cap | immediate initial state, small appends, forced final state |
| Below-cap trickle | isolated writes are immediate; sustained writes are at most 100 ms apart |
| Full-force output after cap | complete turnovers are cap-sized replacements spaced by their encoded size |
| Post-cap trickle | small verified `slide` updates remain responsive; the complete window is not resent |
| Burst then silence | one leading update and one trailing update |
| Error, timeout, or abort | latest bounded state is forced before the error settles |
At a 50 KB cap and 100 KB/s target, repeated complete turnovers settle near two updates per second. Small post-cap slides still use the 100 ms floor.
The rate bound is amortized. An immediate-after-idle update and forced terminal update may each create one cap-bounded burst.
## 5. Capture and spill
`OutputCapture` trims its decoded line-aware working buffer back to twice the byte cap when it exceeds four times the cap. Tail mode drops old text; head mode preserves the original prefix. This amortizes UTF-8 trimming instead of rescanning the retained window for every process chunk.
Spill creation is lazy. Before crossing, the source keeps at most the bounded prefix needed to create a complete archive. On crossing it:
1. pauses stdout and stderr while it creates the file inside the execution environment;
2. opens one persistent append stream with a bounded 8 MB high-water mark;
3. writes the preserved raw prefix and subsequent raw chunks in arrival order;
4. resumes immediately while the writer accepts data;
5. pauses again only when `write()` reports backpressure, then resumes on `drain`.
This avoids both an unbounded promise chain and one async file-open/append cycle per process chunk. A spill create or stream-write failure kills the child and fails execution rather than publishing lossy success.
The spill path is force-published as metadata when it becomes available. Spill writes are awaited before the final output flush.
Node streams remain raw for spill throughput and exact archival bytes. `OutputCapture` uses one streaming `TextDecoder`, so a read boundary cannot split a code point; invalid display control characters are removed only from bounded snapshots, not by scanning the complete raw stream. Line totals count a final unterminated line, and `lastLineBytes` remains exact even when one line exceeds the working buffer.
## 6. Remote execution
When worker and execution environment are colocated, updates are in-process and this publisher primarily bounds capture work. `ToolOutput` remains the downstream event/durability limiter.
When an execution environment is physically remote from its worker, the same bounded updates cross that transport. Full-force output cannot transfer the complete stream: intermediate writes collapse at the source, while the complete stream remains in the source-local spill.
Each real costly boundary gets its own publisher instance. A colocated deployment may bypass transport serialization; a custom tool that bypasses `ExecutionEnv` still passes through the future `ToolOutput` publisher.
## 7. Remaining work
- Move generic custom-tool composition, text retention, event publication, and durable cadence into `ToolOutput`.
- Replace whole `AgentToolResult` progress with Chord operations at that downstream boundary.
- Make memo and output checkpoint persistence one atomic transaction.
- Seed replay from durable output rather than deleting it.
- Put spills in an environment-owned session directory and sweep them after session lifetime plus a crash-retention floor.
- Decide explicit limits for images and structured details; text is bounded, those values are not yet.
- Define raw binary-output behavior. Current shell output remains lossy UTF-8 text.

View File

@@ -0,0 +1,137 @@
diff --git a/packages/agent/src/harness/env/nodejs.ts b/packages/agent/src/harness/env/nodejs.ts
index 732dc84db..db9eb9a57 100644
--- a/packages/agent/src/harness/env/nodejs.ts
+++ b/packages/agent/src/harness/env/nodejs.ts
@@ -19,6 +19,7 @@ import { basename, isAbsolute, join, resolve } from "node:path";
import { createInterface } from "node:readline";
import { fileURLToPath } from "node:url";
import type { Context } from "../context.ts";
+import { OutputCapture } from "../utils/output-capture.ts";
import {
type ExecutionEnv,
ExecutionError,
@@ -28,7 +29,10 @@ import {
type FileKind,
ok,
type Result,
+ type ShellCapabilities,
type ShellExecOptions,
+ type ShellExecResult,
+ type ShellStream,
toError,
} from "../types.ts";
@@ -365,11 +369,15 @@ export class NodeExecutionEnv implements ExecutionEnv {
return ok(join(...parts));
}
+ capabilities(): ShellCapabilities {
+ return { capture: true, spill: true, liveUpdates: true };
+ }
+
async exec(
command: string,
options: ShellExecOptions | undefined,
context: Context,
- ): Promise<Result<{ stdout: string; stderr: string; exitCode: number }, ExecutionError>> {
+ ): Promise<Result<ShellExecResult, ExecutionError>> {
const signal = context.abortSignal;
if (signal?.aborted) return err(new ExecutionError("aborted", "aborted"));
const timeoutMsResult = resolveTimeoutMs(options?.timeout);
@@ -392,9 +400,28 @@ export class NodeExecutionEnv implements ExecutionEnv {
);
}
+ // Spill lives in the exec env, on the machine where the model's own `read`
+ // and `grep` run — the whole reason capping happens here rather than on the
+ // agent side (`executionenv.md` §1).
+ let spillHandle: string | undefined;
+ let spillChain: Promise<void> = Promise.resolve();
+ const wantSpill = options?.capture?.spill === true;
+
return await new Promise((resolvePromise) => {
- let stdout = "";
- let stderr = "";
+ const capture = new OutputCapture(options?.capture, options?.onUpdate);
+ const spillChunk = (chunk: string): Promise<void> => {
+ if (!wantSpill) return Promise.resolve();
+ spillChain = spillChain.then(async () => {
+ if (spillHandle === undefined) {
+ const created = await this.createTempFile({ prefix: "pi-output-", suffix: ".log" }, context);
+ if (!created.ok) return;
+ spillHandle = created.value;
+ capture.setSpillPath(spillHandle);
+ }
+ await this.appendFile(spillHandle, chunk, context);
+ });
+ return spillChain;
+ };
let settled = false;
let timedOut = false;
let callbackError: ExecutionError | undefined;
@@ -407,7 +434,7 @@ export class NodeExecutionEnv implements ExecutionEnv {
}
};
- const settle = (result: Result<{ stdout: string; stderr: string; exitCode: number }, ExecutionError>) => {
+ const settle = (result: Result<ShellExecResult, ExecutionError>) => {
if (timeoutId) clearTimeout(timeoutId);
if (signal) signal.removeEventListener("abort", onAbort);
if (child?.pid) this.activeChildPids.delete(child.pid);
@@ -458,31 +485,32 @@ export class NodeExecutionEnv implements ExecutionEnv {
}
}
+ // setEncoding("utf8") is what keeps a chunk boundary from splitting a
+ // codepoint; without it a multi-byte sequence straddling a read bakes in a
+ // replacement character (`executionenv.md` §8).
child.stdout?.setEncoding("utf8");
child.stderr?.setEncoding("utf8");
- child.stdout?.on("data", (chunk: string) => {
- stdout += chunk;
- try {
- options?.onStdout?.(chunk, context);
- } catch (error) {
- const cause = toError(error);
- callbackError = new ExecutionError("callback_error", cause.message, cause);
- onAbort();
- }
- });
- child.stderr?.on("data", (chunk: string) => {
- stderr += chunk;
+ const feed = (stream: ShellStream, chunk: string) => {
try {
- options?.onStderr?.(chunk, context);
+ capture.push(stream, chunk, context);
+ void spillChunk(chunk);
} catch (error) {
const cause = toError(error);
callbackError = new ExecutionError("callback_error", cause.message, cause);
onAbort();
}
- });
+ };
+ child.stdout?.on("data", (chunk: string) => feed("stdout", chunk));
+ child.stderr?.on("data", (chunk: string) => feed("stderr", chunk));
void waitForChildProcess(child).then(
- (code) => {
+ async (code) => {
+ // The spill chain is async and lazy: the file is created on the first
+ // chunk. Await it before snapshotting, or the returned CapturedOutput
+ // reports no spillPath for output that was in fact archived.
+ await spillChain;
+ capture.flush(context);
+ capture.dispose();
if (callbackError) {
settle(err(callbackError));
return;
@@ -495,7 +523,7 @@ export class NodeExecutionEnv implements ExecutionEnv {
settle(err(new ExecutionError("aborted", "aborted")));
return;
}
- settle(ok({ stdout, stderr, exitCode: code ?? 0 }));
+ settle(ok({ output: capture.snapshot(), exitCode: code ?? 0 }));
},
(error: Error) => settle(err(new ExecutionError("spawn_error", error.message, error))),
);

View File

@@ -0,0 +1,149 @@
diff --git a/packages/agent/test/harness/nodejs-env.test.ts b/packages/agent/test/harness/nodejs-env.test.ts
index 361040495..ee28ef506 100644
--- a/packages/agent/test/harness/nodejs-env.test.ts
+++ b/packages/agent/test/harness/nodejs-env.test.ts
@@ -8,7 +8,6 @@ import { afterEach, describe, expect, it } from "vitest";
import { BACKGROUND_CONTEXT, withAbortSignal } from "../../src/harness/context.ts";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { FileError, getOrThrow } from "../../src/harness/types.ts";
-import { executeShellWithCapture } from "../../src/harness/utils/shell-output.ts";
import { createTempDir } from "./session-test-utils.ts";
const chmodRestorePaths: string[] = [];
@@ -303,7 +302,8 @@ describe("NodeExecutionEnv", () => {
BACKGROUND_CONTEXT,
),
);
- expect(result).toEqual({ stdout: `${await realpath(root)}:ok`, stderr: "", exitCode: 0 });
+ expect(result.output.text).toContain(`${await realpath(root)}:ok`.trim());
+ expect(result.exitCode).toBe(0);
});
it.each([
@@ -334,7 +334,7 @@ describe("NodeExecutionEnv", () => {
),
);
- expect(result.stdout).toBe(`${expectedSessionFile}|true|preserved`);
+ expect(result.output.text).toBe(`${expectedSessionFile}|true|preserved`);
},
);
@@ -358,7 +358,7 @@ describe("NodeExecutionEnv", () => {
),
);
- expect(result.stdout).toBe("::explicit");
+ expect(result.output.text).toBe("::explicit");
} finally {
if (previousInherited === undefined) delete process.env[inheritedKey];
else process.env[inheritedKey] = previousInherited;
@@ -396,7 +396,9 @@ describe("NodeExecutionEnv", () => {
await wslEnv.exec(`name='World'; echo "Hello, ${nameExpansion}!"`, undefined, BACKGROUND_CONTEXT),
);
- expect(result).toEqual({ stdout: "Hello, World!\n", stderr: "args:-s\n", exitCode: 0 });
+ expect(result.output.text).toContain("Hello, World!");
+ expect(result.output.text).toContain("args:-s");
+ expect(result.exitCode).toBe(0);
} finally {
process.chdir(originalCwd);
process.env.PATH = originalPath;
@@ -425,7 +427,7 @@ describe("NodeExecutionEnv", () => {
() => controller.abort(),
),
);
- expect(result.stdout).toContain("child-exiting");
+ expect(result.output.text).toContain("child-exiting");
} finally {
controller.abort();
cleanupDetachedChild(pidFile);
@@ -448,25 +450,30 @@ describe("NodeExecutionEnv", () => {
it("streams stdout and stderr chunks", async () => {
const root = createTempDir();
const env = new NodeExecutionEnv({ cwd: root });
- let stdout = "";
- let stderr = "";
+ // Streams are aggregated in arrival order now; `chunk` carries `stream` so a
+ // renderer can still colour stderr (`executionenv.md` §6).
+ let seen = "";
+ const streams: string[] = [];
const result = getOrThrow(
await env.exec(
"printf out; printf err >&2",
{
- onStdout: (chunk) => {
- stdout += chunk;
- },
- onStderr: (chunk) => {
- stderr += chunk;
+ onUpdate: (update) => {
+ if (update.kind !== "chunk") return;
+ seen += update.text;
+ streams.push(update.stream);
},
},
BACKGROUND_CONTEXT,
),
);
- expect(result).toEqual({ stdout: "out", stderr: "err", exitCode: 0 });
- expect(stdout).toBe("out");
- expect(stderr).toBe("err");
+ expect(result.exitCode).toBe(0);
+ expect(result.output.text).toContain("out");
+ expect(result.output.text).toContain("err");
+ expect(seen).toContain("out");
+ expect(seen).toContain("err");
+ expect(streams).toContain("stdout");
+ expect(streams).toContain("stderr");
});
it("reports a missing working directory before spawning", async () => {
@@ -484,7 +491,8 @@ describe("NodeExecutionEnv", () => {
const root = createTempDir();
const env = new NodeExecutionEnv({ cwd: root });
const result = getOrThrow(await env.exec("exit 7", undefined, BACKGROUND_CONTEXT));
- expect(result).toEqual({ stdout: "", stderr: "", exitCode: 7 });
+ expect(result.output.text).toContain("".trim());
+ expect(result.exitCode).toBe(7);
});
it("returns timeout errors for commands exceeding the timeout", async () => {
@@ -501,7 +509,7 @@ describe("NodeExecutionEnv", () => {
const result = await env.exec(
"printf out",
{
- onStdout: () => {
+ onUpdate: () => {
throw new Error("callback failed");
},
},
@@ -542,12 +550,23 @@ describe("NodeExecutionEnv", () => {
const root = createTempDir();
const env = new NodeExecutionEnv({ cwd: root });
const result = getOrThrow(
- await executeShellWithCapture(env, "yes line | head -n 15000", undefined, BACKGROUND_CONTEXT),
+ await env.exec(
+ "yes line | head -n 15000",
+ {
+ capture: {
+ limits: { maxBytes: 50 * 1024, maxLines: 2000, retain: "tail" },
+ spill: true,
+ },
+ },
+ BACKGROUND_CONTEXT,
+ ),
);
- expect(result.truncated).toBe(true);
- expect(result.fullOutputPath).toBeDefined();
- const fullOutput = getOrThrow(await env.readTextFile(result.fullOutputPath!, BACKGROUND_CONTEXT));
+ expect(result.output.truncation.truncated).toBe(true);
+ // The spill lives inside the exec env, on the machine the command ran on —
+ // which is where the model's own read and grep run (`executionenv.md` §1).
+ expect(result.output.spillPath).toBeDefined();
+ const fullOutput = getOrThrow(await env.readTextFile(result.output.spillPath!, BACKGROUND_CONTEXT));
expect(fullOutput.split("\n").length).toBeGreaterThan(10000);
- expect(result.output.length).toBeLessThan(fullOutput.length);
+ expect(result.output.text.length).toBeLessThan(fullOutput.length);
});
});

View File

@@ -0,0 +1,194 @@
import type { Context } from "../context.ts";
import type { CaptureOptions, CapturedOutput, ShellOutputUpdate, ShellStream } from "../types.ts";
import {
DEFAULT_MAX_BYTES,
DEFAULT_MAX_LINES,
type TruncationResult,
truncateHead,
truncateTail,
utf8ByteLength,
} from "./truncate.ts";
/**
* Bounded capture for a Shell implementation.
*
* Owns the two-stage structure that `shell-output.ts` had inside `bash.ts`: a raw
* buffer trimmed to `maxBytes * 2` as a memory guard, and a retained view
* computed from it per snapshot. `truncateTail` is line-aware — it walks
* backwards line by line and takes a partial only when one line alone exceeds the
* cap — so eviction is not byte arithmetic and cannot be done incrementally.
*
* Emits the three update kinds from `executionenv.md` §4 and enforces the
* ordering invariant: once the cap is crossed, a `chunk` never follows without an
* intervening `snapshot`.
*/
export class OutputCapture {
readonly #maxBytes: number;
readonly #maxLines: number;
readonly #retain: "head" | "tail";
readonly #intervalMs: number;
readonly #onUpdate?: (update: ShellOutputUpdate, context: Context) => void;
/** Raw rolling buffer, held at twice the cap. */
#buffer = "";
/** Everything ever produced. */
#totalBytes = 0;
/** Newlines seen. A trailing partial line is counted separately — a command
* emitting one 60 KB line with no newline has produced one line, not zero. */
#newlines = 0;
#endsWithNewline = true;
/** True once the retained view has begun evicting (tail) or filled (head). */
#crossed = false;
/** Pending chunks for the current coalescing window. */
#pending: { stream: ShellStream; text: string }[] = [];
#lastFlush = 0;
#timer: ReturnType<typeof setTimeout> | undefined;
#spillPath: string | undefined;
constructor(
options: CaptureOptions | undefined,
onUpdate: ((update: ShellOutputUpdate, context: Context) => void) | undefined,
) {
this.#maxBytes = options?.limits.maxBytes ?? DEFAULT_MAX_BYTES;
this.#maxLines = options?.limits.maxLines ?? DEFAULT_MAX_LINES;
this.#retain = options?.limits.retain ?? "tail";
this.#intervalMs = options?.intervalMs ?? 0;
this.#onUpdate = onUpdate;
}
push(stream: ShellStream, text: string, context: Context): void {
if (text === "") return;
this.#totalBytes += utf8ByteLength(text);
this.#newlines += countNewlines(text);
this.#endsWithNewline = text.endsWith("\n");
this.#buffer += text;
// Memory guard. The retained view is computed from this, never equal to it.
const guard = this.#maxBytes * 2;
if (utf8ByteLength(this.#buffer) > guard) {
this.#buffer = trimToLastBytes(this.#buffer, guard);
}
this.#pending.push({ stream, text });
this.#schedule(context);
}
setSpillPath(path: string | undefined): void {
this.#spillPath = path;
}
/** The bounded view, recomputed from the raw buffer. */
snapshot(): CapturedOutput {
const truncation =
this.#retain === "head"
? truncateHead(this.#buffer, { maxBytes: this.#maxBytes, maxLines: this.#maxLines })
: truncateTail(this.#buffer, { maxBytes: this.#maxBytes, maxLines: this.#maxLines });
// The buffer already lost its front, so totals come from counters rather
// than from what survives.
const totalLines = this.#totalLines();
const truncated = this.#totalBytes > this.#maxBytes || totalLines > this.#maxLines;
return {
text: truncation.content,
truncation: {
...truncation,
lastLinePartial: truncation.lastLinePartial,
truncated,
truncatedBy: truncated ? (totalLines > this.#maxLines ? "lines" : "bytes") : null,
totalBytes: this.#totalBytes,
totalLines,
},
...(this.#spillPath === undefined ? {} : { spillPath: this.#spillPath }),
...(truncation.lastLinePartial ? { lastLineBytes: this.#lastLineBytes() } : {}),
};
}
#totalLines(): number {
return this.#newlines + (this.#endsWithNewline || this.#totalBytes === 0 ? 0 : 1);
}
/** Bytes in the final line of the raw buffer, for the single-oversized-line case. */
#lastLineBytes(): number {
const start = this.#buffer.lastIndexOf("\n");
return utf8ByteLength(start === -1 ? this.#buffer : this.#buffer.slice(start + 1));
}
#schedule(context: Context): void {
if (this.#onUpdate === undefined) return;
if (this.#intervalMs === 0) {
this.flush(context);
return;
}
if (this.#timer !== undefined) return;
const wait = Math.max(0, this.#lastFlush + this.#intervalMs - Date.now());
this.#timer = setTimeout(() => {
this.#timer = undefined;
this.flush(context);
}, wait);
}
/** Emit one update for everything buffered since the last flush. */
flush(context: Context): void {
if (this.#onUpdate === undefined) return;
this.#lastFlush = Date.now();
const pending = this.#pending;
this.#pending = [];
if (pending.length === 0) return;
const nowCrossed = this.#totalBytes > this.#maxBytes || this.#totalLines() > this.#maxLines;
if (this.#retain === "head") {
// Head never evicts. Below the cap the appended text is complete; above
// it, only totals move — resending maxBytes to change a number is the
// thing `counters` exists to avoid.
if (!nowCrossed) {
this.#onUpdate({ kind: "chunk", stream: pending[0]!.stream, text: joined(pending) }, context);
} else {
this.#onUpdate({ kind: "counters", truncation: this.snapshot().truncation }, context);
}
this.#crossed = nowCrossed;
return;
}
// Tail. Once the buffer is full every byte in evicts a byte out, so there is
// no cheap-trickle regime past the cap: chunks until full, snapshots after.
if (nowCrossed) {
this.#crossed = true;
this.#onUpdate({ kind: "snapshot", output: this.snapshot() }, context);
return;
}
if (this.#crossed) {
// Defensive: never let a chunk follow the crossing.
this.#onUpdate({ kind: "snapshot", output: this.snapshot() }, context);
return;
}
this.#onUpdate({ kind: "chunk", stream: pending[0]!.stream, text: joined(pending) }, context);
}
dispose(): void {
if (this.#timer !== undefined) clearTimeout(this.#timer);
this.#timer = undefined;
}
}
const joined = (pending: { text: string }[]): string => pending.map((p) => p.text).join("");
function countNewlines(text: string): number {
let count = 0;
for (let index = text.indexOf("\n"); index !== -1; index = text.indexOf("\n", index + 1)) count++;
return count;
}
/** Keep the last `bytes` UTF-8 bytes without splitting a codepoint. */
function trimToLastBytes(text: string, bytes: number): string {
if (utf8ByteLength(text) <= bytes) return text;
let low = 0;
let high = text.length;
while (low < high) {
const mid = Math.floor((low + high) / 2);
if (utf8ByteLength(text.slice(mid)) > bytes) low = mid + 1;
else high = mid;
}
return text.slice(low);
}
export type { TruncationResult };